Sanity tests for OneDrive exports (#3910)
Add sanity tests for OneDrive exports (archive and non-archive exports). --- #### Does this PR need a docs update or release note? - [ ] ✅ Yes, it's included - [ ] 🕐 Yes, but in a later PR - [x] ⛔ No #### Type of change <!--- Please check the type of change your PR introduces: ---> - [ ] 🌻 Feature - [ ] 🐛 Bugfix - [ ] 🗺️ Documentation - [x] 🤖 Supportability/Tests - [ ] 💻 CI/Deployment - [ ] 🧹 Tech Debt/Cleanup #### Issue(s) <!-- Can reference multiple issues. Use one of the following "magic words" - "closes, fixes" to auto-close the Github issue. --> * https://github.com/alcionai/corso/issues/3889 #### Test Plan <!-- How will this be tested prior to merging.--> - [ ] 💪 Manual - [ ] ⚡ Unit test - [x] 💚 E2E
This commit is contained in:
parent
26d5083269
commit
8f30db4f6e
72
.github/actions/backup-restore-test/action.yml
vendored
72
.github/actions/backup-restore-test/action.yml
vendored
@ -72,12 +72,78 @@ runs:
|
|||||||
|
|
||||||
cat /tmp/corsologs
|
cat /tmp/corsologs
|
||||||
|
|
||||||
- name: Check ${{ inputs.service }} ${{ inputs.kind }}
|
- name: Check restore ${{ inputs.service }} ${{ inputs.kind }}
|
||||||
shell: bash
|
shell: bash
|
||||||
working-directory: src
|
working-directory: src
|
||||||
env:
|
env:
|
||||||
SANITY_RESTORE_FOLDER: ${{ steps.restore.outputs.result }}
|
SANITY_TEST_KIND: restore
|
||||||
SANITY_RESTORE_SERVICE: ${{ inputs.service }}
|
SANITY_TEST_FOLDER: ${{ steps.restore.outputs.result }}
|
||||||
|
SANITY_TEST_SERVICE: ${{ inputs.service }}
|
||||||
|
TEST_DATA: ${{ inputs.test-folder }}
|
||||||
|
BASE_BACKUP: ${{ inputs.base-backup }}
|
||||||
|
run: |
|
||||||
|
CORSO_LOG_FILE=${{ inputs.log-dir }}/gotest-validate-${{ inputs.service }}-${{inputs.kind }}.log
|
||||||
|
./sanity-test
|
||||||
|
|
||||||
|
- name: Export ${{ inputs.service }} ${{ inputs.kind }}
|
||||||
|
id: export
|
||||||
|
shell: bash
|
||||||
|
working-directory: src
|
||||||
|
if: ${{ inputs.service == 'onedrive' }} # Export only available for OneDrive
|
||||||
|
run: |
|
||||||
|
set -euo pipefail
|
||||||
|
CORSO_LOG_FILE=${{ inputs.log-dir }}/gotest-restore-${{ inputs.service }}-${{inputs.kind }}.log
|
||||||
|
./corso export '${{ inputs.service }}' \
|
||||||
|
/tmp/export-${{ inputs.service }}-${{inputs.kind }} \
|
||||||
|
--no-stats \
|
||||||
|
--hide-progress \
|
||||||
|
${{ inputs.export-args }} \
|
||||||
|
--backup '${{ steps.backup.outputs.result }}'
|
||||||
|
|
||||||
|
cat /tmp/corsologs
|
||||||
|
|
||||||
|
- name: Check export ${{ inputs.service }} ${{ inputs.kind }}
|
||||||
|
shell: bash
|
||||||
|
working-directory: src
|
||||||
|
if: ${{ inputs.service == 'onedrive' }}
|
||||||
|
env:
|
||||||
|
SANITY_TEST_KIND: export
|
||||||
|
SANITY_TEST_FOLDER: /tmp/export-${{ inputs.service }}-${{inputs.kind }}
|
||||||
|
SANITY_TEST_SERVICE: ${{ inputs.service }}
|
||||||
|
TEST_DATA: ${{ inputs.test-folder }}
|
||||||
|
BASE_BACKUP: ${{ inputs.base-backup }}
|
||||||
|
run: |
|
||||||
|
CORSO_LOG_FILE=${{ inputs.log-dir }}/gotest-validate-${{ inputs.service }}-${{inputs.kind }}.log
|
||||||
|
./sanity-test
|
||||||
|
|
||||||
|
- name: Export archive ${{ inputs.service }} ${{ inputs.kind }}
|
||||||
|
id: export-archive
|
||||||
|
shell: bash
|
||||||
|
working-directory: src
|
||||||
|
if: ${{ inputs.service == 'onedrive' }} # Export only available for OneDrive
|
||||||
|
run: |
|
||||||
|
set -euo pipefail
|
||||||
|
CORSO_LOG_FILE=${{ inputs.log-dir }}/gotest-restore-${{ inputs.service }}-${{inputs.kind }}.log
|
||||||
|
./corso export '${{ inputs.service }}' \
|
||||||
|
/tmp/export-${{ inputs.service }}-${{inputs.kind }}-archive \
|
||||||
|
--no-stats \
|
||||||
|
--hide-progress \
|
||||||
|
--archive \
|
||||||
|
${{ inputs.export-args }} \
|
||||||
|
--backup '${{ steps.backup.outputs.result }}'
|
||||||
|
|
||||||
|
unzip /tmp/export-${{ inputs.service }}-${{inputs.kind }}-archive/*.zip \
|
||||||
|
-d /tmp/export-${{ inputs.service }}-${{inputs.kind }}-unzipped
|
||||||
|
cat /tmp/corsologs
|
||||||
|
|
||||||
|
- name: Check archive export ${{ inputs.service }} ${{ inputs.kind }}
|
||||||
|
shell: bash
|
||||||
|
working-directory: src
|
||||||
|
if: ${{ inputs.service == 'onedrive' }}
|
||||||
|
env:
|
||||||
|
SANITY_TEST_KIND: export
|
||||||
|
SANITY_TEST_FOLDER: /tmp/export-${{ inputs.service }}-${{inputs.kind }}-unzipped
|
||||||
|
SANITY_TEST_SERVICE: ${{ inputs.service }}
|
||||||
TEST_DATA: ${{ inputs.test-folder }}
|
TEST_DATA: ${{ inputs.test-folder }}
|
||||||
BASE_BACKUP: ${{ inputs.base-backup }}
|
BASE_BACKUP: ${{ inputs.base-backup }}
|
||||||
run: |
|
run: |
|
||||||
|
|||||||
6
src/cmd/sanity_test/common/common.go
Normal file
6
src/cmd/sanity_test/common/common.go
Normal file
@ -0,0 +1,6 @@
|
|||||||
|
package common
|
||||||
|
|
||||||
|
type PermissionInfo struct {
|
||||||
|
EntityID string
|
||||||
|
Roles []string
|
||||||
|
}
|
||||||
82
src/cmd/sanity_test/common/utils.go
Normal file
82
src/cmd/sanity_test/common/utils.go
Normal file
@ -0,0 +1,82 @@
|
|||||||
|
package common
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/alcionai/corso/src/internal/common/dttm"
|
||||||
|
"github.com/alcionai/corso/src/pkg/logger"
|
||||||
|
)
|
||||||
|
|
||||||
|
func Assert(
|
||||||
|
ctx context.Context,
|
||||||
|
passes func() bool,
|
||||||
|
header string,
|
||||||
|
expect, current any,
|
||||||
|
) {
|
||||||
|
if passes() {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
header = "Error: " + header
|
||||||
|
expected := fmt.Sprintf("* Expected: %+v", expect)
|
||||||
|
got := fmt.Sprintf("* Current: %+v", current)
|
||||||
|
|
||||||
|
logger.Ctx(ctx).Info(strings.Join([]string{header, expected, got}, " "))
|
||||||
|
|
||||||
|
fmt.Println(header)
|
||||||
|
fmt.Println(expected)
|
||||||
|
fmt.Println(got)
|
||||||
|
|
||||||
|
os.Exit(1)
|
||||||
|
}
|
||||||
|
|
||||||
|
func Fatal(ctx context.Context, msg string, err error) {
|
||||||
|
logger.CtxErr(ctx, err).Error("test failure: " + msg)
|
||||||
|
fmt.Println(msg+": ", err)
|
||||||
|
os.Exit(1)
|
||||||
|
}
|
||||||
|
|
||||||
|
func MustGetTimeFromName(ctx context.Context, name string) (time.Time, bool) {
|
||||||
|
t, err := dttm.ExtractTime(name)
|
||||||
|
if err != nil && !errors.Is(err, dttm.ErrNoTimeString) {
|
||||||
|
Fatal(ctx, "extracting time from name: "+name, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return t, !errors.Is(err, dttm.ErrNoTimeString)
|
||||||
|
}
|
||||||
|
|
||||||
|
func IsWithinTimeBound(ctx context.Context, bound, check time.Time, hasTime bool) bool {
|
||||||
|
if hasTime {
|
||||||
|
if bound.Before(check) {
|
||||||
|
logger.Ctx(ctx).
|
||||||
|
With("boundary_time", bound, "check_time", check).
|
||||||
|
Info("skipping restore folder: not older than time bound")
|
||||||
|
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
|
||||||
|
func FilterSlice(sl []string, remove string) []string {
|
||||||
|
r := []string{}
|
||||||
|
|
||||||
|
for _, s := range sl {
|
||||||
|
if !strings.EqualFold(s, remove) {
|
||||||
|
r = append(r, s)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return r
|
||||||
|
}
|
||||||
|
|
||||||
|
func LogAndPrint(ctx context.Context, tmpl string, vs ...any) {
|
||||||
|
logger.Ctx(ctx).Infof(tmpl, vs...)
|
||||||
|
fmt.Printf(tmpl+"\n", vs...)
|
||||||
|
}
|
||||||
88
src/cmd/sanity_test/export/onedrive.go
Normal file
88
src/cmd/sanity_test/export/onedrive.go
Normal file
@ -0,0 +1,88 @@
|
|||||||
|
package export
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/alcionai/clues"
|
||||||
|
msgraphsdk "github.com/microsoftgraph/msgraph-sdk-go"
|
||||||
|
|
||||||
|
"github.com/alcionai/corso/src/cmd/sanity_test/common"
|
||||||
|
"github.com/alcionai/corso/src/cmd/sanity_test/restore"
|
||||||
|
"github.com/alcionai/corso/src/internal/common/ptr"
|
||||||
|
)
|
||||||
|
|
||||||
|
func CheckOneDriveExport(
|
||||||
|
ctx context.Context,
|
||||||
|
client *msgraphsdk.GraphServiceClient,
|
||||||
|
userID, folderName, dataFolder string,
|
||||||
|
) {
|
||||||
|
drive, err := client.
|
||||||
|
Users().
|
||||||
|
ByUserId(userID).
|
||||||
|
Drive().
|
||||||
|
Get(ctx, nil)
|
||||||
|
if err != nil {
|
||||||
|
common.Fatal(ctx, "getting the drive:", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// map itemID -> item size
|
||||||
|
var (
|
||||||
|
fileSizes = make(map[string]int64)
|
||||||
|
exportFileSizes = make(map[string]int64)
|
||||||
|
startTime = time.Now()
|
||||||
|
)
|
||||||
|
|
||||||
|
err = filepath.Walk(folderName, func(path string, info os.FileInfo, err error) error {
|
||||||
|
if err != nil {
|
||||||
|
return clues.Stack(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if info.IsDir() {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
relPath, err := filepath.Rel(folderName, path)
|
||||||
|
if err != nil {
|
||||||
|
return clues.Stack(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
exportFileSizes[relPath] = info.Size()
|
||||||
|
if startTime.After(info.ModTime()) {
|
||||||
|
startTime = info.ModTime()
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
fmt.Println("Error walking the path:", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
_ = restore.PopulateDriveDetails(
|
||||||
|
ctx,
|
||||||
|
client,
|
||||||
|
ptr.Val(drive.GetId()),
|
||||||
|
folderName,
|
||||||
|
dataFolder,
|
||||||
|
fileSizes,
|
||||||
|
map[string][]common.PermissionInfo{},
|
||||||
|
startTime)
|
||||||
|
|
||||||
|
for fileName, expected := range fileSizes {
|
||||||
|
common.LogAndPrint(ctx, "checking for file: %s", fileName)
|
||||||
|
|
||||||
|
got := exportFileSizes[fileName]
|
||||||
|
|
||||||
|
common.Assert(
|
||||||
|
ctx,
|
||||||
|
func() bool { return expected == got },
|
||||||
|
fmt.Sprintf("different file size: %s", fileName),
|
||||||
|
expected,
|
||||||
|
got)
|
||||||
|
}
|
||||||
|
|
||||||
|
fmt.Println("Success")
|
||||||
|
}
|
||||||
@ -12,7 +12,7 @@ import (
|
|||||||
"github.com/microsoftgraph/msgraph-sdk-go/models"
|
"github.com/microsoftgraph/msgraph-sdk-go/models"
|
||||||
"github.com/microsoftgraph/msgraph-sdk-go/users"
|
"github.com/microsoftgraph/msgraph-sdk-go/users"
|
||||||
|
|
||||||
"github.com/alcionai/corso/src/cmd/sanity_test/utils"
|
"github.com/alcionai/corso/src/cmd/sanity_test/common"
|
||||||
"github.com/alcionai/corso/src/internal/common/ptr"
|
"github.com/alcionai/corso/src/internal/common/ptr"
|
||||||
"github.com/alcionai/corso/src/pkg/filters"
|
"github.com/alcionai/corso/src/pkg/filters"
|
||||||
)
|
)
|
||||||
@ -35,7 +35,7 @@ func CheckEmailRestoration(
|
|||||||
for {
|
for {
|
||||||
result, err := builder.Get(ctx, nil)
|
result, err := builder.Get(ctx, nil)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
utils.Fatal(ctx, "getting mail folders", err)
|
common.Fatal(ctx, "getting mail folders", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
values := result.GetValue()
|
values := result.GetValue()
|
||||||
@ -79,7 +79,7 @@ func CheckEmailRestoration(
|
|||||||
ChildFolders().
|
ChildFolders().
|
||||||
Get(ctx, nil)
|
Get(ctx, nil)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
utils.Fatal(ctx, "getting restore folder child folders", err)
|
common.Fatal(ctx, "getting restore folder child folders", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, fld := range childFolder.GetValue() {
|
for _, fld := range childFolder.GetValue() {
|
||||||
@ -102,7 +102,7 @@ func verifyEmailData(ctx context.Context, restoreMessageCount, messageCount map[
|
|||||||
for fldName, expected := range messageCount {
|
for fldName, expected := range messageCount {
|
||||||
got := restoreMessageCount[fldName]
|
got := restoreMessageCount[fldName]
|
||||||
|
|
||||||
utils.Assert(
|
common.Assert(
|
||||||
ctx,
|
ctx,
|
||||||
func() bool { return expected == got },
|
func() bool { return expected == got },
|
||||||
fmt.Sprintf("Restore item counts do not match: %s", fldName),
|
fmt.Sprintf("Restore item counts do not match: %s", fldName),
|
||||||
@ -142,7 +142,7 @@ func getAllMailSubFolders(
|
|||||||
ChildFolders().
|
ChildFolders().
|
||||||
Get(ctx, options)
|
Get(ctx, options)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
utils.Fatal(ctx, "getting mail subfolders", err)
|
common.Fatal(ctx, "getting mail subfolders", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, child := range childFolder.GetValue() {
|
for _, child := range childFolder.GetValue() {
|
||||||
@ -194,7 +194,7 @@ func checkAllSubFolder(
|
|||||||
ChildFolders().
|
ChildFolders().
|
||||||
Get(ctx, options)
|
Get(ctx, options)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
utils.Fatal(ctx, "getting mail subfolders", err)
|
common.Fatal(ctx, "getting mail subfolders", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, child := range childFolder.GetValue() {
|
for _, child := range childFolder.GetValue() {
|
||||||
|
|||||||
@ -10,7 +10,7 @@ import (
|
|||||||
msgraphsdk "github.com/microsoftgraph/msgraph-sdk-go"
|
msgraphsdk "github.com/microsoftgraph/msgraph-sdk-go"
|
||||||
"golang.org/x/exp/slices"
|
"golang.org/x/exp/slices"
|
||||||
|
|
||||||
"github.com/alcionai/corso/src/cmd/sanity_test/utils"
|
"github.com/alcionai/corso/src/cmd/sanity_test/common"
|
||||||
"github.com/alcionai/corso/src/internal/common/ptr"
|
"github.com/alcionai/corso/src/internal/common/ptr"
|
||||||
"github.com/alcionai/corso/src/pkg/path"
|
"github.com/alcionai/corso/src/pkg/path"
|
||||||
)
|
)
|
||||||
@ -19,11 +19,6 @@ const (
|
|||||||
owner = "owner"
|
owner = "owner"
|
||||||
)
|
)
|
||||||
|
|
||||||
type permissionInfo struct {
|
|
||||||
entityID string
|
|
||||||
roles []string
|
|
||||||
}
|
|
||||||
|
|
||||||
func CheckOneDriveRestoration(
|
func CheckOneDriveRestoration(
|
||||||
ctx context.Context,
|
ctx context.Context,
|
||||||
client *msgraphsdk.GraphServiceClient,
|
client *msgraphsdk.GraphServiceClient,
|
||||||
@ -36,7 +31,7 @@ func CheckOneDriveRestoration(
|
|||||||
Drive().
|
Drive().
|
||||||
Get(ctx, nil)
|
Get(ctx, nil)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
utils.Fatal(ctx, "getting the drive:", err)
|
common.Fatal(ctx, "getting the drive:", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
checkDriveRestoration(
|
checkDriveRestoration(
|
||||||
@ -66,15 +61,58 @@ func checkDriveRestoration(
|
|||||||
// map itemID -> item size
|
// map itemID -> item size
|
||||||
fileSizes = make(map[string]int64)
|
fileSizes = make(map[string]int64)
|
||||||
// map itemID -> permission id -> []permission roles
|
// map itemID -> permission id -> []permission roles
|
||||||
folderPermissions = make(map[string][]permissionInfo)
|
folderPermissions = make(map[string][]common.PermissionInfo)
|
||||||
restoreFile = make(map[string]int64)
|
restoreFile = make(map[string]int64)
|
||||||
restoredFolderPermissions = make(map[string][]permissionInfo)
|
restoredFolderPermissions = make(map[string][]common.PermissionInfo)
|
||||||
)
|
)
|
||||||
|
|
||||||
var restoreFolderID string
|
|
||||||
|
|
||||||
ctx = clues.Add(ctx, "drive_id", driveID, "drive_name", driveName)
|
ctx = clues.Add(ctx, "drive_id", driveID, "drive_name", driveName)
|
||||||
|
|
||||||
|
restoreFolderID := PopulateDriveDetails(
|
||||||
|
ctx,
|
||||||
|
client,
|
||||||
|
driveID,
|
||||||
|
folderName,
|
||||||
|
dataFolder,
|
||||||
|
fileSizes,
|
||||||
|
folderPermissions,
|
||||||
|
startTime)
|
||||||
|
|
||||||
|
getRestoredDrive(ctx, client, driveID, restoreFolderID, restoreFile, restoredFolderPermissions, startTime)
|
||||||
|
|
||||||
|
checkRestoredDriveItemPermissions(
|
||||||
|
ctx,
|
||||||
|
service,
|
||||||
|
skipPermissionTest,
|
||||||
|
folderPermissions,
|
||||||
|
restoredFolderPermissions)
|
||||||
|
|
||||||
|
for fileName, expected := range fileSizes {
|
||||||
|
common.LogAndPrint(ctx, "checking for file: %s", fileName)
|
||||||
|
|
||||||
|
got := restoreFile[fileName]
|
||||||
|
|
||||||
|
common.Assert(
|
||||||
|
ctx,
|
||||||
|
func() bool { return expected == got },
|
||||||
|
fmt.Sprintf("different file size: %s", fileName),
|
||||||
|
expected,
|
||||||
|
got)
|
||||||
|
}
|
||||||
|
|
||||||
|
fmt.Println("Success")
|
||||||
|
}
|
||||||
|
|
||||||
|
func PopulateDriveDetails(
|
||||||
|
ctx context.Context,
|
||||||
|
client *msgraphsdk.GraphServiceClient,
|
||||||
|
driveID, folderName, dataFolder string,
|
||||||
|
fileSizes map[string]int64,
|
||||||
|
folderPermissions map[string][]common.PermissionInfo,
|
||||||
|
startTime time.Time,
|
||||||
|
) string {
|
||||||
|
var restoreFolderID string
|
||||||
|
|
||||||
response, err := client.
|
response, err := client.
|
||||||
Drives().
|
Drives().
|
||||||
ByDriveId(driveID).
|
ByDriveId(driveID).
|
||||||
@ -83,7 +121,7 @@ func checkDriveRestoration(
|
|||||||
Children().
|
Children().
|
||||||
Get(ctx, nil)
|
Get(ctx, nil)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
utils.Fatal(ctx, "getting drive by id", err)
|
common.Fatal(ctx, "getting drive by id", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, driveItem := range response.GetValue() {
|
for _, driveItem := range response.GetValue() {
|
||||||
@ -98,7 +136,7 @@ func checkDriveRestoration(
|
|||||||
}
|
}
|
||||||
|
|
||||||
if itemName != dataFolder {
|
if itemName != dataFolder {
|
||||||
utils.LogAndPrint(ctx, "test data for folder: %s", dataFolder)
|
common.LogAndPrint(ctx, "test data for folder: %s", dataFolder)
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -114,7 +152,7 @@ func checkDriveRestoration(
|
|||||||
// currently we don't restore blank folders.
|
// currently we don't restore blank folders.
|
||||||
// skip permission check for empty folders
|
// skip permission check for empty folders
|
||||||
if ptr.Val(driveItem.GetFolder().GetChildCount()) == 0 {
|
if ptr.Val(driveItem.GetFolder().GetChildCount()) == 0 {
|
||||||
utils.LogAndPrint(ctx, "skipped empty folder: %s", itemName)
|
common.LogAndPrint(ctx, "skipped empty folder: %s", itemName)
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -122,37 +160,15 @@ func checkDriveRestoration(
|
|||||||
getOneDriveChildFolder(ctx, client, driveID, itemID, itemName, fileSizes, folderPermissions, startTime)
|
getOneDriveChildFolder(ctx, client, driveID, itemID, itemName, fileSizes, folderPermissions, startTime)
|
||||||
}
|
}
|
||||||
|
|
||||||
getRestoredDrive(ctx, client, driveID, restoreFolderID, restoreFile, restoredFolderPermissions, startTime)
|
return restoreFolderID
|
||||||
|
|
||||||
checkRestoredDriveItemPermissions(
|
|
||||||
ctx,
|
|
||||||
service,
|
|
||||||
skipPermissionTest,
|
|
||||||
folderPermissions,
|
|
||||||
restoredFolderPermissions)
|
|
||||||
|
|
||||||
for fileName, expected := range fileSizes {
|
|
||||||
utils.LogAndPrint(ctx, "checking for file: %s", fileName)
|
|
||||||
|
|
||||||
got := restoreFile[fileName]
|
|
||||||
|
|
||||||
utils.Assert(
|
|
||||||
ctx,
|
|
||||||
func() bool { return expected == got },
|
|
||||||
fmt.Sprintf("different file size: %s", fileName),
|
|
||||||
expected,
|
|
||||||
got)
|
|
||||||
}
|
|
||||||
|
|
||||||
fmt.Println("Success")
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func checkRestoredDriveItemPermissions(
|
func checkRestoredDriveItemPermissions(
|
||||||
ctx context.Context,
|
ctx context.Context,
|
||||||
service path.ServiceType,
|
service path.ServiceType,
|
||||||
skip bool,
|
skip bool,
|
||||||
folderPermissions map[string][]permissionInfo,
|
folderPermissions map[string][]common.PermissionInfo,
|
||||||
restoredFolderPermissions map[string][]permissionInfo,
|
restoredFolderPermissions map[string][]common.PermissionInfo,
|
||||||
) {
|
) {
|
||||||
if skip {
|
if skip {
|
||||||
return
|
return
|
||||||
@ -164,12 +180,12 @@ func checkRestoredDriveItemPermissions(
|
|||||||
**/
|
**/
|
||||||
|
|
||||||
for folderName, permissions := range folderPermissions {
|
for folderName, permissions := range folderPermissions {
|
||||||
utils.LogAndPrint(ctx, "checking for folder: %s", folderName)
|
common.LogAndPrint(ctx, "checking for folder: %s", folderName)
|
||||||
|
|
||||||
restoreFolderPerm := restoredFolderPermissions[folderName]
|
restoreFolderPerm := restoredFolderPermissions[folderName]
|
||||||
|
|
||||||
if len(permissions) < 1 {
|
if len(permissions) < 1 {
|
||||||
utils.LogAndPrint(ctx, "no permissions found in: %s", folderName)
|
common.LogAndPrint(ctx, "no permissions found in: %s", folderName)
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -179,7 +195,7 @@ func checkRestoredDriveItemPermissions(
|
|||||||
permCheck = func() bool { return len(permissions) <= len(restoreFolderPerm) }
|
permCheck = func() bool { return len(permissions) <= len(restoreFolderPerm) }
|
||||||
}
|
}
|
||||||
|
|
||||||
utils.Assert(
|
common.Assert(
|
||||||
ctx,
|
ctx,
|
||||||
permCheck,
|
permCheck,
|
||||||
fmt.Sprintf("wrong number of restored permissions: %s", folderName),
|
fmt.Sprintf("wrong number of restored permissions: %s", folderName),
|
||||||
@ -187,25 +203,25 @@ func checkRestoredDriveItemPermissions(
|
|||||||
restoreFolderPerm)
|
restoreFolderPerm)
|
||||||
|
|
||||||
for _, perm := range permissions {
|
for _, perm := range permissions {
|
||||||
eqID := func(pi permissionInfo) bool { return strings.EqualFold(pi.entityID, perm.entityID) }
|
eqID := func(pi common.PermissionInfo) bool { return strings.EqualFold(pi.EntityID, perm.EntityID) }
|
||||||
i := slices.IndexFunc(restoreFolderPerm, eqID)
|
i := slices.IndexFunc(restoreFolderPerm, eqID)
|
||||||
|
|
||||||
utils.Assert(
|
common.Assert(
|
||||||
ctx,
|
ctx,
|
||||||
func() bool { return i >= 0 },
|
func() bool { return i >= 0 },
|
||||||
fmt.Sprintf("permission was restored in: %s", folderName),
|
fmt.Sprintf("permission was restored in: %s", folderName),
|
||||||
perm.entityID,
|
perm.EntityID,
|
||||||
restoreFolderPerm)
|
restoreFolderPerm)
|
||||||
|
|
||||||
// permissions should be sorted, so a by-index comparison works
|
// permissions should be sorted, so a by-index comparison works
|
||||||
restored := restoreFolderPerm[i]
|
restored := restoreFolderPerm[i]
|
||||||
|
|
||||||
utils.Assert(
|
common.Assert(
|
||||||
ctx,
|
ctx,
|
||||||
func() bool { return slices.Equal(perm.roles, restored.roles) },
|
func() bool { return slices.Equal(perm.Roles, restored.Roles) },
|
||||||
fmt.Sprintf("different roles restored: %s", folderName),
|
fmt.Sprintf("different roles restored: %s", folderName),
|
||||||
perm.roles,
|
perm.Roles,
|
||||||
restored.roles)
|
restored.Roles)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -215,12 +231,12 @@ func getOneDriveChildFolder(
|
|||||||
client *msgraphsdk.GraphServiceClient,
|
client *msgraphsdk.GraphServiceClient,
|
||||||
driveID, itemID, parentName string,
|
driveID, itemID, parentName string,
|
||||||
fileSizes map[string]int64,
|
fileSizes map[string]int64,
|
||||||
folderPermission map[string][]permissionInfo,
|
folderPermission map[string][]common.PermissionInfo,
|
||||||
startTime time.Time,
|
startTime time.Time,
|
||||||
) {
|
) {
|
||||||
response, err := client.Drives().ByDriveId(driveID).Items().ByDriveItemId(itemID).Children().Get(ctx, nil)
|
response, err := client.Drives().ByDriveId(driveID).Items().ByDriveItemId(itemID).Children().Get(ctx, nil)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
utils.Fatal(ctx, "getting child folder", err)
|
common.Fatal(ctx, "getting child folder", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, driveItem := range response.GetValue() {
|
for _, driveItem := range response.GetValue() {
|
||||||
@ -230,8 +246,8 @@ func getOneDriveChildFolder(
|
|||||||
fullName = parentName + "/" + itemName
|
fullName = parentName + "/" + itemName
|
||||||
)
|
)
|
||||||
|
|
||||||
folderTime, hasTime := utils.MustGetTimeFromName(ctx, itemName)
|
folderTime, hasTime := common.MustGetTimeFromName(ctx, itemName)
|
||||||
if !utils.IsWithinTimeBound(ctx, startTime, folderTime, hasTime) {
|
if !common.IsWithinTimeBound(ctx, startTime, folderTime, hasTime) {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -247,7 +263,7 @@ func getOneDriveChildFolder(
|
|||||||
// currently we don't restore blank folders.
|
// currently we don't restore blank folders.
|
||||||
// skip permission check for empty folders
|
// skip permission check for empty folders
|
||||||
if ptr.Val(driveItem.GetFolder().GetChildCount()) == 0 {
|
if ptr.Val(driveItem.GetFolder().GetChildCount()) == 0 {
|
||||||
utils.LogAndPrint(ctx, "skipped empty folder: %s", fullName)
|
common.LogAndPrint(ctx, "skipped empty folder: %s", fullName)
|
||||||
|
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
@ -262,7 +278,7 @@ func getRestoredDrive(
|
|||||||
client *msgraphsdk.GraphServiceClient,
|
client *msgraphsdk.GraphServiceClient,
|
||||||
driveID, restoreFolderID string,
|
driveID, restoreFolderID string,
|
||||||
restoreFile map[string]int64,
|
restoreFile map[string]int64,
|
||||||
restoreFolder map[string][]permissionInfo,
|
restoreFolder map[string][]common.PermissionInfo,
|
||||||
startTime time.Time,
|
startTime time.Time,
|
||||||
) {
|
) {
|
||||||
restored, err := client.
|
restored, err := client.
|
||||||
@ -273,7 +289,7 @@ func getRestoredDrive(
|
|||||||
Children().
|
Children().
|
||||||
Get(ctx, nil)
|
Get(ctx, nil)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
utils.Fatal(ctx, "getting child folder", err)
|
common.Fatal(ctx, "getting child folder", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, item := range restored.GetValue() {
|
for _, item := range restored.GetValue() {
|
||||||
@ -305,8 +321,8 @@ func permissionIn(
|
|||||||
ctx context.Context,
|
ctx context.Context,
|
||||||
client *msgraphsdk.GraphServiceClient,
|
client *msgraphsdk.GraphServiceClient,
|
||||||
driveID, itemID string,
|
driveID, itemID string,
|
||||||
) []permissionInfo {
|
) []common.PermissionInfo {
|
||||||
pi := []permissionInfo{}
|
pi := []common.PermissionInfo{}
|
||||||
|
|
||||||
pcr, err := client.
|
pcr, err := client.
|
||||||
Drives().
|
Drives().
|
||||||
@ -316,7 +332,7 @@ func permissionIn(
|
|||||||
Permissions().
|
Permissions().
|
||||||
Get(ctx, nil)
|
Get(ctx, nil)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
utils.Fatal(ctx, "getting permission", err)
|
common.Fatal(ctx, "getting permission", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, perm := range pcr.GetValue() {
|
for _, perm := range pcr.GetValue() {
|
||||||
@ -326,7 +342,7 @@ func permissionIn(
|
|||||||
|
|
||||||
var (
|
var (
|
||||||
gv2 = perm.GetGrantedToV2()
|
gv2 = perm.GetGrantedToV2()
|
||||||
permInfo = permissionInfo{}
|
permInfo = common.PermissionInfo{}
|
||||||
entityID string
|
entityID string
|
||||||
)
|
)
|
||||||
|
|
||||||
@ -337,14 +353,14 @@ func permissionIn(
|
|||||||
entityID = ptr.Val(gv2.GetGroup().GetId())
|
entityID = ptr.Val(gv2.GetGroup().GetId())
|
||||||
}
|
}
|
||||||
|
|
||||||
roles := utils.FilterSlice(perm.GetRoles(), owner)
|
roles := common.FilterSlice(perm.GetRoles(), owner)
|
||||||
for _, role := range roles {
|
for _, role := range roles {
|
||||||
permInfo.entityID = entityID
|
permInfo.EntityID = entityID
|
||||||
permInfo.roles = append(permInfo.roles, role)
|
permInfo.Roles = append(permInfo.Roles, role)
|
||||||
}
|
}
|
||||||
|
|
||||||
if len(roles) > 0 {
|
if len(roles) > 0 {
|
||||||
slices.Sort(permInfo.roles)
|
slices.Sort(permInfo.Roles)
|
||||||
pi = append(pi, permInfo)
|
pi = append(pi, permInfo)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@ -6,7 +6,7 @@ import (
|
|||||||
|
|
||||||
msgraphsdk "github.com/microsoftgraph/msgraph-sdk-go"
|
msgraphsdk "github.com/microsoftgraph/msgraph-sdk-go"
|
||||||
|
|
||||||
"github.com/alcionai/corso/src/cmd/sanity_test/utils"
|
"github.com/alcionai/corso/src/cmd/sanity_test/common"
|
||||||
"github.com/alcionai/corso/src/internal/common/ptr"
|
"github.com/alcionai/corso/src/internal/common/ptr"
|
||||||
"github.com/alcionai/corso/src/pkg/path"
|
"github.com/alcionai/corso/src/pkg/path"
|
||||||
)
|
)
|
||||||
@ -23,7 +23,7 @@ func CheckSharePointRestoration(
|
|||||||
Drive().
|
Drive().
|
||||||
Get(ctx, nil)
|
Get(ctx, nil)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
utils.Fatal(ctx, "getting the drive:", err)
|
common.Fatal(ctx, "getting the drive:", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
checkDriveRestoration(
|
checkDriveRestoration(
|
||||||
|
|||||||
@ -9,8 +9,9 @@ import (
|
|||||||
"github.com/alcionai/clues"
|
"github.com/alcionai/clues"
|
||||||
msgraphsdk "github.com/microsoftgraph/msgraph-sdk-go"
|
msgraphsdk "github.com/microsoftgraph/msgraph-sdk-go"
|
||||||
|
|
||||||
|
"github.com/alcionai/corso/src/cmd/sanity_test/common"
|
||||||
|
"github.com/alcionai/corso/src/cmd/sanity_test/export"
|
||||||
"github.com/alcionai/corso/src/cmd/sanity_test/restore"
|
"github.com/alcionai/corso/src/cmd/sanity_test/restore"
|
||||||
"github.com/alcionai/corso/src/cmd/sanity_test/utils"
|
|
||||||
"github.com/alcionai/corso/src/internal/m365/graph"
|
"github.com/alcionai/corso/src/internal/m365/graph"
|
||||||
"github.com/alcionai/corso/src/internal/tester/tconfig"
|
"github.com/alcionai/corso/src/internal/tester/tconfig"
|
||||||
"github.com/alcionai/corso/src/pkg/logger"
|
"github.com/alcionai/corso/src/pkg/logger"
|
||||||
@ -35,16 +36,16 @@ func main() {
|
|||||||
os.Getenv("AZURE_CLIENT_ID"),
|
os.Getenv("AZURE_CLIENT_ID"),
|
||||||
os.Getenv("AZURE_CLIENT_SECRET"))
|
os.Getenv("AZURE_CLIENT_SECRET"))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
utils.Fatal(ctx, "creating adapter", err)
|
common.Fatal(ctx, "creating adapter", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
var (
|
var (
|
||||||
client = msgraphsdk.NewGraphServiceClient(adapter)
|
client = msgraphsdk.NewGraphServiceClient(adapter)
|
||||||
testUser = tconfig.GetM365UserID(ctx)
|
testUser = tconfig.GetM365UserID(ctx)
|
||||||
testSite = tconfig.GetM365SiteID(ctx)
|
testSite = tconfig.GetM365SiteID(ctx)
|
||||||
testService = os.Getenv("SANITY_RESTORE_SERVICE")
|
testKind = os.Getenv("SANITY_TEST_KIND") // restore or export (cli arg?)
|
||||||
folder = strings.TrimSpace(os.Getenv("SANITY_RESTORE_FOLDER"))
|
testService = os.Getenv("SANITY_TEST_SERVICE")
|
||||||
startTime, _ = utils.MustGetTimeFromName(ctx, folder)
|
folder = strings.TrimSpace(os.Getenv("SANITY_TEST_FOLDER"))
|
||||||
dataFolder = os.Getenv("TEST_DATA")
|
dataFolder = os.Getenv("TEST_DATA")
|
||||||
baseBackupFolder = os.Getenv("BASE_BACKUP")
|
baseBackupFolder = os.Getenv("BASE_BACKUP")
|
||||||
)
|
)
|
||||||
@ -53,19 +54,33 @@ func main() {
|
|||||||
ctx,
|
ctx,
|
||||||
"resource_owner", testUser,
|
"resource_owner", testUser,
|
||||||
"service", testService,
|
"service", testService,
|
||||||
"sanity_restore_folder", folder,
|
"sanity_restore_folder", folder)
|
||||||
"start_time", startTime.Format(time.RFC3339Nano))
|
|
||||||
|
|
||||||
logger.Ctx(ctx).Info("starting sanity test check")
|
logger.Ctx(ctx).Info("starting sanity test check")
|
||||||
|
|
||||||
switch testService {
|
switch testKind {
|
||||||
case "exchange":
|
case "restore":
|
||||||
restore.CheckEmailRestoration(ctx, client, testUser, folder, dataFolder, baseBackupFolder, startTime)
|
startTime, _ := common.MustGetTimeFromName(ctx, folder)
|
||||||
case "onedrive":
|
clues.Add(ctx, "sanity_restore_start_time", startTime.Format(time.RFC3339))
|
||||||
restore.CheckOneDriveRestoration(ctx, client, testUser, folder, dataFolder, startTime)
|
|
||||||
case "sharepoint":
|
switch testService {
|
||||||
restore.CheckSharePointRestoration(ctx, client, testSite, testUser, folder, dataFolder, startTime)
|
case "exchange":
|
||||||
|
restore.CheckEmailRestoration(ctx, client, testUser, folder, dataFolder, baseBackupFolder, startTime)
|
||||||
|
case "onedrive":
|
||||||
|
restore.CheckOneDriveRestoration(ctx, client, testUser, folder, dataFolder, startTime)
|
||||||
|
case "sharepoint":
|
||||||
|
restore.CheckSharePointRestoration(ctx, client, testSite, testUser, folder, dataFolder, startTime)
|
||||||
|
default:
|
||||||
|
common.Fatal(ctx, "unknown service for restore sanity tests", nil)
|
||||||
|
}
|
||||||
|
case "export":
|
||||||
|
switch testService {
|
||||||
|
case "onedrive":
|
||||||
|
export.CheckOneDriveExport(ctx, client, testUser, folder, dataFolder)
|
||||||
|
default:
|
||||||
|
common.Fatal(ctx, "unknown service for export sanity tests", nil)
|
||||||
|
}
|
||||||
default:
|
default:
|
||||||
utils.Fatal(ctx, "unknown service for restore sanity tests", nil)
|
common.Fatal(ctx, "unknown test kind (expected restore or export)", nil)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user