adds local sanity tests for lists (#5031)
adds local sanity tests for lists for reference: - `lists` do not have `folders` concept, they are directly created in a site. As a result cannot be restored to a different folder to test. Therefore they need to be created in adjacent to the 'backed-up' lists. Due to this there is a requirement of `SANITY_TEST_RESTORE_CONTAINER_PREFIX`. Using this, during restore check, this env-var is used to find restored lists. - Similarly, `SANITY_TEST_SOURCE_CONTAINER` is also used a prefix matcher to filter created test lists - When exported, lists are stored in local into the provided directory as `<SANITY_TEST_RESTORE_CONTAINER>/Lists/<list-id>/<list-id>.json` should go in after: - https://github.com/alcionai/corso/pull/5024 - https://github.com/alcionai/corso/pull/5026 #### Does this PR need a docs update or release note? - [x] ⛔ No #### Type of change <!--- Please check the type of change your PR introduces: ---> - [x] 🤖 Supportability/Tests #### Issue(s) #4754 #### Test Plan <!-- How will this be tested prior to merging.--> - [x] 💪 Manual - [x] ⚡ Unit test - [x] 💚 E2E
This commit is contained in:
Hitesh Pattanayak
GitHub
parent
dfb4a73f56
commit
07da7f16a1
@ -19,33 +19,40 @@ type PermissionInfo struct {
|
||||
}
|
||||
|
||||
const (
|
||||
sanityBackupID = "SANITY_BACKUP_ID"
|
||||
sanityTestSourceContainer = "SANITY_TEST_SOURCE_CONTAINER"
|
||||
sanityTestRestoreContainer = "SANITY_TEST_RESTORE_CONTAINER"
|
||||
sanityTestUser = "SANITY_TEST_USER"
|
||||
sanityBackupID = "SANITY_BACKUP_ID"
|
||||
sanityTestSourceContainer = "SANITY_TEST_SOURCE_CONTAINER"
|
||||
sanityTestRestoreContainer = "SANITY_TEST_RESTORE_CONTAINER"
|
||||
sanityTestRestoreContainerPrefix = "SANITY_TEST_RESTORE_CONTAINER_PREFIX"
|
||||
sanityTestUser = "SANITY_TEST_USER"
|
||||
sanityTestCategory = "SANITY_TEST_CATEGORY"
|
||||
)
|
||||
|
||||
type Envs struct {
|
||||
BackupID string
|
||||
SourceContainer string
|
||||
RestoreContainer string
|
||||
GroupID string
|
||||
SiteID string
|
||||
UserID string
|
||||
TeamSiteID string
|
||||
// applies for sharepoint lists only
|
||||
RestoreContainerPrefix string
|
||||
Category string
|
||||
GroupID string
|
||||
SiteID string
|
||||
UserID string
|
||||
TeamSiteID string
|
||||
}
|
||||
|
||||
func EnvVars(ctx context.Context) Envs {
|
||||
folder := strings.TrimSpace(os.Getenv(sanityTestRestoreContainer))
|
||||
|
||||
e := Envs{
|
||||
BackupID: os.Getenv(sanityBackupID),
|
||||
SourceContainer: os.Getenv(sanityTestSourceContainer),
|
||||
RestoreContainer: folder,
|
||||
GroupID: tconfig.GetM365TeamID(ctx),
|
||||
SiteID: tconfig.GetM365SiteID(ctx),
|
||||
UserID: tconfig.GetM365UserID(ctx),
|
||||
TeamSiteID: tconfig.GetM365TeamSiteID(ctx),
|
||||
BackupID: os.Getenv(sanityBackupID),
|
||||
SourceContainer: os.Getenv(sanityTestSourceContainer),
|
||||
RestoreContainer: folder,
|
||||
Category: os.Getenv(sanityTestCategory),
|
||||
RestoreContainerPrefix: os.Getenv(sanityTestRestoreContainerPrefix),
|
||||
GroupID: tconfig.GetM365TeamID(ctx),
|
||||
SiteID: tconfig.GetM365SiteID(ctx),
|
||||
UserID: tconfig.GetM365UserID(ctx),
|
||||
TeamSiteID: tconfig.GetM365TeamSiteID(ctx),
|
||||
}
|
||||
|
||||
if len(os.Getenv(sanityTestUser)) > 0 {
|
||||
|
||||
@ -20,31 +20,18 @@ func BuildFilepathSanitree(
|
||||
info os.FileInfo,
|
||||
err error,
|
||||
) error {
|
||||
if err != nil {
|
||||
Fatal(ctx, "error passed to filepath walker", err)
|
||||
}
|
||||
|
||||
relPath, err := filepath.Rel(rootDir, p)
|
||||
if err != nil {
|
||||
Fatal(ctx, "getting relative filepath", err)
|
||||
}
|
||||
|
||||
if info != nil {
|
||||
Debugf(ctx, "adding: %s", relPath)
|
||||
}
|
||||
|
||||
if root == nil {
|
||||
root = &Sanitree[fs.FileInfo, fs.FileInfo]{
|
||||
Self: info,
|
||||
ID: info.Name(),
|
||||
Name: info.Name(),
|
||||
Leaves: map[string]*Sanileaf[fs.FileInfo, fs.FileInfo]{},
|
||||
Children: map[string]*Sanitree[fs.FileInfo, fs.FileInfo]{},
|
||||
}
|
||||
|
||||
root = CreateNewRoot(info, true)
|
||||
return nil
|
||||
}
|
||||
|
||||
relPath := GetRelativePath(
|
||||
ctx,
|
||||
rootDir,
|
||||
p,
|
||||
info,
|
||||
err)
|
||||
|
||||
elems := path.Split(relPath)
|
||||
node := root.NodeAt(ctx, elems[:len(elems)-1])
|
||||
|
||||
@ -78,3 +65,41 @@ func BuildFilepathSanitree(
|
||||
|
||||
return root
|
||||
}
|
||||
|
||||
func CreateNewRoot(info fs.FileInfo, initChildren bool) *Sanitree[fs.FileInfo, fs.FileInfo] {
|
||||
root := &Sanitree[fs.FileInfo, fs.FileInfo]{
|
||||
Self: info,
|
||||
ID: info.Name(),
|
||||
Name: info.Name(),
|
||||
Leaves: map[string]*Sanileaf[fs.FileInfo, fs.FileInfo]{},
|
||||
Children: map[string]*Sanitree[fs.FileInfo, fs.FileInfo]{},
|
||||
}
|
||||
|
||||
if initChildren {
|
||||
root.Children = map[string]*Sanitree[fs.FileInfo, fs.FileInfo]{}
|
||||
}
|
||||
|
||||
return root
|
||||
}
|
||||
|
||||
func GetRelativePath(
|
||||
ctx context.Context,
|
||||
rootDir, p string,
|
||||
info fs.FileInfo,
|
||||
fileWalkerErr error,
|
||||
) string {
|
||||
if fileWalkerErr != nil {
|
||||
Fatal(ctx, "error passed to filepath walker", fileWalkerErr)
|
||||
}
|
||||
|
||||
relPath, err := filepath.Rel(rootDir, p)
|
||||
if err != nil {
|
||||
Fatal(ctx, "getting relative filepath", err)
|
||||
}
|
||||
|
||||
if info != nil {
|
||||
Debugf(ctx, "adding: %s", relPath)
|
||||
}
|
||||
|
||||
return relPath
|
||||
}
|
||||
|
||||
@ -2,10 +2,20 @@ package export
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"github.com/alcionai/clues"
|
||||
"github.com/microsoftgraph/msgraph-sdk-go/models"
|
||||
"github.com/tidwall/gjson"
|
||||
|
||||
"github.com/alcionai/corso/src/cmd/sanity_test/common"
|
||||
"github.com/alcionai/corso/src/cmd/sanity_test/driveish"
|
||||
"github.com/alcionai/corso/src/cmd/sanity_test/restore"
|
||||
"github.com/alcionai/corso/src/pkg/path"
|
||||
"github.com/alcionai/corso/src/pkg/services/m365/api"
|
||||
)
|
||||
|
||||
@ -14,15 +24,128 @@ func CheckSharePointExport(
|
||||
ac api.Client,
|
||||
envs common.Envs,
|
||||
) {
|
||||
drive, err := ac.Sites().GetDefaultDrive(ctx, envs.SiteID)
|
||||
if err != nil {
|
||||
common.Fatal(ctx, "getting the drive:", err)
|
||||
if envs.Category == path.ListsCategory.String() {
|
||||
CheckSharepointListsExport(ctx, ac, envs)
|
||||
}
|
||||
|
||||
envs.RestoreContainer = filepath.Join(envs.RestoreContainer, "Libraries/Documents") // check in default loc
|
||||
driveish.CheckExport(
|
||||
ctx,
|
||||
ac,
|
||||
drive,
|
||||
envs)
|
||||
if envs.Category == path.LibrariesCategory.String() {
|
||||
drive, err := ac.Sites().GetDefaultDrive(ctx, envs.SiteID)
|
||||
if err != nil {
|
||||
common.Fatal(ctx, "getting the drive:", err)
|
||||
}
|
||||
|
||||
envs.RestoreContainer = filepath.Join(envs.RestoreContainer, "Libraries/Documents") // check in default loc
|
||||
driveish.CheckExport(
|
||||
ctx,
|
||||
ac,
|
||||
drive,
|
||||
envs)
|
||||
}
|
||||
}
|
||||
|
||||
func CheckSharepointListsExport(
|
||||
ctx context.Context,
|
||||
ac api.Client,
|
||||
envs common.Envs,
|
||||
) {
|
||||
exportFolderName := path.ListsCategory.HumanString()
|
||||
|
||||
sourceTree := restore.BuildListsSanitree(ctx, ac, envs.SiteID, envs.SourceContainer, exportFolderName)
|
||||
|
||||
listsExportDir := filepath.Join(envs.RestoreContainer, exportFolderName)
|
||||
exportedTree := BuildFilepathSanitreeForSharepointLists(ctx, listsExportDir)
|
||||
|
||||
ctx = clues.Add(
|
||||
ctx,
|
||||
"export_container_id", exportedTree.ID,
|
||||
"export_container_name", exportedTree.Name,
|
||||
"source_container_id", sourceTree.ID,
|
||||
"source_container_name", sourceTree.Name)
|
||||
|
||||
comparator := func(
|
||||
ctx context.Context,
|
||||
expect *common.Sanitree[models.Siteable, models.Listable],
|
||||
result *common.Sanitree[fs.FileInfo, fs.FileInfo],
|
||||
) {
|
||||
modifiedResultLeaves := map[string]*common.Sanileaf[fs.FileInfo, fs.FileInfo]{}
|
||||
|
||||
for key, val := range result.Leaves {
|
||||
fixedName := strings.TrimSuffix(key, ".json")
|
||||
|
||||
modifiedResultLeaves[fixedName] = val
|
||||
}
|
||||
|
||||
common.CompareLeaves(ctx, expect.Leaves, modifiedResultLeaves, nil)
|
||||
}
|
||||
|
||||
common.CompareDiffTrees(
|
||||
ctx,
|
||||
sourceTree,
|
||||
exportedTree,
|
||||
comparator)
|
||||
|
||||
common.Infof(ctx, "Success")
|
||||
}
|
||||
|
||||
func BuildFilepathSanitreeForSharepointLists(
|
||||
ctx context.Context,
|
||||
rootDir string,
|
||||
) *common.Sanitree[fs.FileInfo, fs.FileInfo] {
|
||||
var root *common.Sanitree[fs.FileInfo, fs.FileInfo]
|
||||
|
||||
walker := func(
|
||||
p string,
|
||||
info os.FileInfo,
|
||||
err error,
|
||||
) error {
|
||||
if root == nil {
|
||||
root = common.CreateNewRoot(info, false)
|
||||
return nil
|
||||
}
|
||||
|
||||
relPath := common.GetRelativePath(
|
||||
ctx,
|
||||
rootDir,
|
||||
p,
|
||||
info,
|
||||
err)
|
||||
|
||||
if !info.IsDir() {
|
||||
file, err := os.Open(p)
|
||||
if err != nil {
|
||||
common.Fatal(ctx, "opening file to read", err)
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
content, err := io.ReadAll(file)
|
||||
if err != nil {
|
||||
common.Fatal(ctx, "reading file", err)
|
||||
}
|
||||
|
||||
res := gjson.Get(string(content), "items.#")
|
||||
itemsCount := res.Num
|
||||
|
||||
elems := path.Split(relPath)
|
||||
|
||||
node := root.NodeAt(ctx, elems[:len(elems)-2])
|
||||
node.CountLeaves++
|
||||
node.Leaves[info.Name()] = &common.Sanileaf[fs.FileInfo, fs.FileInfo]{
|
||||
Parent: node,
|
||||
Self: info,
|
||||
ID: info.Name(),
|
||||
Name: info.Name(),
|
||||
// using list item count as size for lists
|
||||
Size: int64(itemsCount),
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
err := filepath.Walk(rootDir, walker)
|
||||
if err != nil {
|
||||
common.Fatal(ctx, "walking filepath", err)
|
||||
}
|
||||
|
||||
return root
|
||||
}
|
||||
|
||||
@ -2,9 +2,15 @@ package restore
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"github.com/alcionai/clues"
|
||||
"github.com/microsoftgraph/msgraph-sdk-go/models"
|
||||
|
||||
"github.com/alcionai/corso/src/cmd/sanity_test/common"
|
||||
"github.com/alcionai/corso/src/cmd/sanity_test/driveish"
|
||||
"github.com/alcionai/corso/src/internal/common/ptr"
|
||||
"github.com/alcionai/corso/src/pkg/services/m365/api"
|
||||
)
|
||||
|
||||
@ -13,16 +19,155 @@ func CheckSharePointRestoration(
|
||||
ac api.Client,
|
||||
envs common.Envs,
|
||||
) {
|
||||
drive, err := ac.Sites().GetDefaultDrive(ctx, envs.SiteID)
|
||||
if err != nil {
|
||||
common.Fatal(ctx, "getting site's default drive:", err)
|
||||
if envs.Category == "lists" {
|
||||
CheckSharePointListsRestoration(ctx, ac, envs)
|
||||
}
|
||||
|
||||
driveish.CheckRestoration(
|
||||
ctx,
|
||||
ac,
|
||||
drive,
|
||||
envs,
|
||||
// skip permissions tests
|
||||
nil)
|
||||
if envs.Category == "libraries" {
|
||||
drive, err := ac.Sites().GetDefaultDrive(ctx, envs.SiteID)
|
||||
if err != nil {
|
||||
common.Fatal(ctx, "getting site's default drive:", err)
|
||||
}
|
||||
|
||||
driveish.CheckRestoration(
|
||||
ctx,
|
||||
ac,
|
||||
drive,
|
||||
envs,
|
||||
// skip permissions tests
|
||||
nil)
|
||||
}
|
||||
}
|
||||
|
||||
func CheckSharePointListsRestoration(
|
||||
ctx context.Context,
|
||||
ac api.Client,
|
||||
envs common.Envs,
|
||||
) {
|
||||
restoredTree := BuildListsSanitree(ctx, ac, envs.SiteID, envs.RestoreContainerPrefix, "")
|
||||
sourceTree := BuildListsSanitree(ctx, ac, envs.SiteID, envs.SourceContainer, "")
|
||||
|
||||
ctx = clues.Add(
|
||||
ctx,
|
||||
"restore_container_id", restoredTree.ID,
|
||||
"restore_container_name", restoredTree.Name,
|
||||
"source_container_id", sourceTree.ID,
|
||||
"source_container_name", sourceTree.Name)
|
||||
|
||||
common.CompareDiffTrees[models.Siteable, models.Listable](
|
||||
ctx,
|
||||
sourceTree,
|
||||
restoredTree,
|
||||
nil)
|
||||
|
||||
common.Infof(ctx, "Success")
|
||||
}
|
||||
|
||||
func BuildListsSanitree(
|
||||
ctx context.Context,
|
||||
ac api.Client,
|
||||
siteID string,
|
||||
restoreContainerPrefix, exportFolderName string,
|
||||
) *common.Sanitree[models.Siteable, models.Listable] {
|
||||
common.Infof(ctx, "building sanitree for lists of site: %s", siteID)
|
||||
|
||||
site, err := ac.Sites().GetByID(ctx, siteID, api.CallConfig{})
|
||||
if err != nil {
|
||||
common.Fatal(
|
||||
ctx,
|
||||
fmt.Sprintf("finding site by id %q", siteID),
|
||||
err)
|
||||
}
|
||||
|
||||
cfg := api.CallConfig{
|
||||
Select: idAnd("displayName", "list", "lastModifiedDateTime"),
|
||||
}
|
||||
|
||||
lists, err := ac.Lists().GetLists(ctx, siteID, cfg)
|
||||
if err != nil {
|
||||
common.Fatal(
|
||||
ctx,
|
||||
fmt.Sprintf("finding lists of site with id %q", siteID),
|
||||
err)
|
||||
}
|
||||
|
||||
lists = filterToSupportedLists(lists)
|
||||
|
||||
filteredLists := filterListsByPrefix(lists, restoreContainerPrefix)
|
||||
|
||||
rootTreeName := ptr.Val(site.GetDisplayName())
|
||||
// lists get stored into the local dir at destination/Lists/
|
||||
if len(exportFolderName) > 0 {
|
||||
rootTreeName = exportFolderName
|
||||
}
|
||||
|
||||
root := &common.Sanitree[models.Siteable, models.Listable]{
|
||||
Self: site,
|
||||
ID: ptr.Val(site.GetId()),
|
||||
Name: rootTreeName,
|
||||
CountLeaves: len(filteredLists),
|
||||
Leaves: map[string]*common.Sanileaf[models.Siteable, models.Listable]{},
|
||||
}
|
||||
|
||||
for _, list := range filteredLists {
|
||||
listID := ptr.Val(list.GetId())
|
||||
|
||||
listItems, err := ac.Lists().GetListItems(ctx, siteID, listID, api.CallConfig{})
|
||||
if err != nil {
|
||||
common.Fatal(
|
||||
ctx,
|
||||
fmt.Sprintf("finding listItems of list with id %q", listID),
|
||||
err)
|
||||
}
|
||||
|
||||
m := &common.Sanileaf[models.Siteable, models.Listable]{
|
||||
Parent: root,
|
||||
Self: list,
|
||||
ID: listID,
|
||||
Name: ptr.Val(list.GetDisplayName()),
|
||||
// using list item count as size for lists
|
||||
Size: int64(len(listItems)),
|
||||
}
|
||||
|
||||
root.Leaves[m.ID] = m
|
||||
}
|
||||
|
||||
return root
|
||||
}
|
||||
|
||||
func filterToSupportedLists(lists []models.Listable) []models.Listable {
|
||||
filteredLists := make([]models.Listable, 0)
|
||||
|
||||
for _, list := range lists {
|
||||
if !api.SkipListTemplates.HasKey(ptr.Val(list.GetList().GetTemplate())) {
|
||||
filteredLists = append(filteredLists, list)
|
||||
}
|
||||
}
|
||||
|
||||
return filteredLists
|
||||
}
|
||||
|
||||
func filterListsByPrefix(lists []models.Listable, prefix string) []models.Listable {
|
||||
result := []models.Listable{}
|
||||
|
||||
for _, list := range lists {
|
||||
for _, pfx := range strings.Split(prefix, ",") {
|
||||
if strings.HasPrefix(ptr.Val(list.GetDisplayName()), pfx) {
|
||||
result = append(result, list)
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
func idAnd(ss ...string) []string {
|
||||
id := []string{"id"}
|
||||
|
||||
if len(ss) == 0 {
|
||||
return id
|
||||
}
|
||||
|
||||
return append(id, ss...)
|
||||
}
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user