adds local sanity tests for lists (#5031)

adds local sanity tests for lists

for reference:
- `lists` do not have `folders` concept, they are directly created in a site. As a result cannot be restored to a different folder to test. Therefore they need to be created in adjacent to the 'backed-up' lists. Due to this there is a requirement of `SANITY_TEST_RESTORE_CONTAINER_PREFIX`. Using this, during restore check, this env-var is used to find restored lists.
- Similarly, `SANITY_TEST_SOURCE_CONTAINER` is also used a prefix matcher to filter created test lists
- When exported, lists are stored in local into the provided directory as `<SANITY_TEST_RESTORE_CONTAINER>/Lists/<list-id>/<list-id>.json`

should go in after:
- https://github.com/alcionai/corso/pull/5024
- https://github.com/alcionai/corso/pull/5026

#### Does this PR need a docs update or release note?
- [x] ⛔ No

#### Type of change

<!--- Please check the type of change your PR introduces: --->
- [x] 🤖 Supportability/Tests

#### Issue(s)
#4754

#### Test Plan

<!-- How will this be tested prior to merging.-->
- [x] 💪 Manual
- [x] ⚡ Unit test
- [x] 💚 E2E

src/cmd/sanity_test/common/common.go

@@ -19,33 +19,40 @@ type PermissionInfo struct {
 }
 
 const (
-	sanityBackupID             = "SANITY_BACKUP_ID"
+	sanityBackupID                   = "SANITY_BACKUP_ID"
-	sanityTestSourceContainer  = "SANITY_TEST_SOURCE_CONTAINER"
+	sanityTestSourceContainer        = "SANITY_TEST_SOURCE_CONTAINER"
-	sanityTestRestoreContainer = "SANITY_TEST_RESTORE_CONTAINER"
+	sanityTestRestoreContainer       = "SANITY_TEST_RESTORE_CONTAINER"
-	sanityTestUser             = "SANITY_TEST_USER"
+	sanityTestRestoreContainerPrefix = "SANITY_TEST_RESTORE_CONTAINER_PREFIX"
+	sanityTestUser                   = "SANITY_TEST_USER"
+	sanityTestCategory               = "SANITY_TEST_CATEGORY"
 )
 
 type Envs struct {
 	BackupID         string
 	SourceContainer  string
 	RestoreContainer string
-	GroupID          string
+	// applies for sharepoint lists only
-	SiteID           string
+	RestoreContainerPrefix string
-	UserID           string
+	Category               string
-	TeamSiteID       string
+	GroupID                string
+	SiteID                 string
+	UserID                 string
+	TeamSiteID             string
 }
 
 func EnvVars(ctx context.Context) Envs {
 	folder := strings.TrimSpace(os.Getenv(sanityTestRestoreContainer))
 
 	e := Envs{
-		BackupID:         os.Getenv(sanityBackupID),
+		BackupID:               os.Getenv(sanityBackupID),
-		SourceContainer:  os.Getenv(sanityTestSourceContainer),
+		SourceContainer:        os.Getenv(sanityTestSourceContainer),
-		RestoreContainer: folder,
+		RestoreContainer:       folder,
-		GroupID:          tconfig.GetM365TeamID(ctx),
+		Category:               os.Getenv(sanityTestCategory),
-		SiteID:           tconfig.GetM365SiteID(ctx),
+		RestoreContainerPrefix: os.Getenv(sanityTestRestoreContainerPrefix),
-		UserID:           tconfig.GetM365UserID(ctx),
+		GroupID:                tconfig.GetM365TeamID(ctx),
-		TeamSiteID:       tconfig.GetM365TeamSiteID(ctx),
+		SiteID:                 tconfig.GetM365SiteID(ctx),
+		UserID:                 tconfig.GetM365UserID(ctx),
+		TeamSiteID:             tconfig.GetM365TeamSiteID(ctx),
 	}
 
 	if len(os.Getenv(sanityTestUser)) > 0 {

src/cmd/sanity_test/common/filepath.go

@@ -20,31 +20,18 @@ func BuildFilepathSanitree(
 		info os.FileInfo,
 		err error,
 	) error {
-		if err != nil {
-			Fatal(ctx, "error passed to filepath walker", err)
-		}
-
-		relPath, err := filepath.Rel(rootDir, p)
-		if err != nil {
-			Fatal(ctx, "getting relative filepath", err)
-		}
-
-		if info != nil {
-			Debugf(ctx, "adding: %s", relPath)
-		}
-
 		if root == nil {
-			root = &Sanitree[fs.FileInfo, fs.FileInfo]{
+			root = CreateNewRoot(info, true)
-				Self:     info,
-				ID:       info.Name(),
-				Name:     info.Name(),
-				Leaves:   map[string]*Sanileaf[fs.FileInfo, fs.FileInfo]{},
-				Children: map[string]*Sanitree[fs.FileInfo, fs.FileInfo]{},
-			}
-
 			return nil
 		}
 
+		relPath := GetRelativePath(
+			ctx,
+			rootDir,
+			p,
+			info,
+			err)
+
 		elems := path.Split(relPath)
 		node := root.NodeAt(ctx, elems[:len(elems)-1])
 
@@ -78,3 +65,41 @@ func BuildFilepathSanitree(
 
 	return root
 }
+
+func CreateNewRoot(info fs.FileInfo, initChildren bool) *Sanitree[fs.FileInfo, fs.FileInfo] {
+	root := &Sanitree[fs.FileInfo, fs.FileInfo]{
+		Self:     info,
+		ID:       info.Name(),
+		Name:     info.Name(),
+		Leaves:   map[string]*Sanileaf[fs.FileInfo, fs.FileInfo]{},
+		Children: map[string]*Sanitree[fs.FileInfo, fs.FileInfo]{},
+	}
+
+	if initChildren {
+		root.Children = map[string]*Sanitree[fs.FileInfo, fs.FileInfo]{}
+	}
+
+	return root
+}
+
+func GetRelativePath(
+	ctx context.Context,
+	rootDir, p string,
+	info fs.FileInfo,
+	fileWalkerErr error,
+) string {
+	if fileWalkerErr != nil {
+		Fatal(ctx, "error passed to filepath walker", fileWalkerErr)
+	}
+
+	relPath, err := filepath.Rel(rootDir, p)
+	if err != nil {
+		Fatal(ctx, "getting relative filepath", err)
+	}
+
+	if info != nil {
+		Debugf(ctx, "adding: %s", relPath)
+	}
+
+	return relPath
+}

src/cmd/sanity_test/export/sharepoint.go

@@ -2,10 +2,20 @@ package export
 
 import (
 	"context"
+	"io"
+	"io/fs"
+	"os"
 	"path/filepath"
+	"strings"
+
+	"github.com/alcionai/clues"
+	"github.com/microsoftgraph/msgraph-sdk-go/models"
+	"github.com/tidwall/gjson"
 
 	"github.com/alcionai/corso/src/cmd/sanity_test/common"
 	"github.com/alcionai/corso/src/cmd/sanity_test/driveish"
+	"github.com/alcionai/corso/src/cmd/sanity_test/restore"
+	"github.com/alcionai/corso/src/pkg/path"
 	"github.com/alcionai/corso/src/pkg/services/m365/api"
 )
 
@@ -14,15 +24,128 @@ func CheckSharePointExport(
 	ac api.Client,
 	envs common.Envs,
 ) {
-	drive, err := ac.Sites().GetDefaultDrive(ctx, envs.SiteID)
+	if envs.Category == path.ListsCategory.String() {
-	if err != nil {
+		CheckSharepointListsExport(ctx, ac, envs)
-		common.Fatal(ctx, "getting the drive:", err)
 	}
 
-	envs.RestoreContainer = filepath.Join(envs.RestoreContainer, "Libraries/Documents") // check in default loc
+	if envs.Category == path.LibrariesCategory.String() {
-	driveish.CheckExport(
+		drive, err := ac.Sites().GetDefaultDrive(ctx, envs.SiteID)
-		ctx,
+		if err != nil {
-		ac,
+			common.Fatal(ctx, "getting the drive:", err)
-		drive,
+		}
-		envs)
+
+		envs.RestoreContainer = filepath.Join(envs.RestoreContainer, "Libraries/Documents") // check in default loc
+		driveish.CheckExport(
+			ctx,
+			ac,
+			drive,
+			envs)
+	}
+}
+
+func CheckSharepointListsExport(
+	ctx context.Context,
+	ac api.Client,
+	envs common.Envs,
+) {
+	exportFolderName := path.ListsCategory.HumanString()
+
+	sourceTree := restore.BuildListsSanitree(ctx, ac, envs.SiteID, envs.SourceContainer, exportFolderName)
+
+	listsExportDir := filepath.Join(envs.RestoreContainer, exportFolderName)
+	exportedTree := BuildFilepathSanitreeForSharepointLists(ctx, listsExportDir)
+
+	ctx = clues.Add(
+		ctx,
+		"export_container_id", exportedTree.ID,
+		"export_container_name", exportedTree.Name,
+		"source_container_id", sourceTree.ID,
+		"source_container_name", sourceTree.Name)
+
+	comparator := func(
+		ctx context.Context,
+		expect *common.Sanitree[models.Siteable, models.Listable],
+		result *common.Sanitree[fs.FileInfo, fs.FileInfo],
+	) {
+		modifiedResultLeaves := map[string]*common.Sanileaf[fs.FileInfo, fs.FileInfo]{}
+
+		for key, val := range result.Leaves {
+			fixedName := strings.TrimSuffix(key, ".json")
+
+			modifiedResultLeaves[fixedName] = val
+		}
+
+		common.CompareLeaves(ctx, expect.Leaves, modifiedResultLeaves, nil)
+	}
+
+	common.CompareDiffTrees(
+		ctx,
+		sourceTree,
+		exportedTree,
+		comparator)
+
+	common.Infof(ctx, "Success")
+}
+
+func BuildFilepathSanitreeForSharepointLists(
+	ctx context.Context,
+	rootDir string,
+) *common.Sanitree[fs.FileInfo, fs.FileInfo] {
+	var root *common.Sanitree[fs.FileInfo, fs.FileInfo]
+
+	walker := func(
+		p string,
+		info os.FileInfo,
+		err error,
+	) error {
+		if root == nil {
+			root = common.CreateNewRoot(info, false)
+			return nil
+		}
+
+		relPath := common.GetRelativePath(
+			ctx,
+			rootDir,
+			p,
+			info,
+			err)
+
+		if !info.IsDir() {
+			file, err := os.Open(p)
+			if err != nil {
+				common.Fatal(ctx, "opening file to read", err)
+			}
+			defer file.Close()
+
+			content, err := io.ReadAll(file)
+			if err != nil {
+				common.Fatal(ctx, "reading file", err)
+			}
+
+			res := gjson.Get(string(content), "items.#")
+			itemsCount := res.Num
+
+			elems := path.Split(relPath)
+
+			node := root.NodeAt(ctx, elems[:len(elems)-2])
+			node.CountLeaves++
+			node.Leaves[info.Name()] = &common.Sanileaf[fs.FileInfo, fs.FileInfo]{
+				Parent: node,
+				Self:   info,
+				ID:     info.Name(),
+				Name:   info.Name(),
+				// using list item count as size for lists
+				Size: int64(itemsCount),
+			}
+		}
+
+		return nil
+	}
+
+	err := filepath.Walk(rootDir, walker)
+	if err != nil {
+		common.Fatal(ctx, "walking filepath", err)
+	}
+
+	return root
 }

src/cmd/sanity_test/restore/sharepoint.go

@@ -2,9 +2,15 @@ package restore
 
 import (
 	"context"
+	"fmt"
+	"strings"
+
+	"github.com/alcionai/clues"
+	"github.com/microsoftgraph/msgraph-sdk-go/models"
 
 	"github.com/alcionai/corso/src/cmd/sanity_test/common"
 	"github.com/alcionai/corso/src/cmd/sanity_test/driveish"
+	"github.com/alcionai/corso/src/internal/common/ptr"
 	"github.com/alcionai/corso/src/pkg/services/m365/api"
 )
 
@@ -13,16 +19,155 @@ func CheckSharePointRestoration(
 	ac api.Client,
 	envs common.Envs,
 ) {
-	drive, err := ac.Sites().GetDefaultDrive(ctx, envs.SiteID)
+	if envs.Category == "lists" {
-	if err != nil {
+		CheckSharePointListsRestoration(ctx, ac, envs)
-		common.Fatal(ctx, "getting site's default drive:", err)
 	}
 
-	driveish.CheckRestoration(
+	if envs.Category == "libraries" {
-		ctx,
+		drive, err := ac.Sites().GetDefaultDrive(ctx, envs.SiteID)
-		ac,
+		if err != nil {
-		drive,
+			common.Fatal(ctx, "getting site's default drive:", err)
-		envs,
+		}
-		// skip permissions tests
+
-		nil)
+		driveish.CheckRestoration(
+			ctx,
+			ac,
+			drive,
+			envs,
+			// skip permissions tests
+			nil)
+	}
+}
+
+func CheckSharePointListsRestoration(
+	ctx context.Context,
+	ac api.Client,
+	envs common.Envs,
+) {
+	restoredTree := BuildListsSanitree(ctx, ac, envs.SiteID, envs.RestoreContainerPrefix, "")
+	sourceTree := BuildListsSanitree(ctx, ac, envs.SiteID, envs.SourceContainer, "")
+
+	ctx = clues.Add(
+		ctx,
+		"restore_container_id", restoredTree.ID,
+		"restore_container_name", restoredTree.Name,
+		"source_container_id", sourceTree.ID,
+		"source_container_name", sourceTree.Name)
+
+	common.CompareDiffTrees[models.Siteable, models.Listable](
+		ctx,
+		sourceTree,
+		restoredTree,
+		nil)
+
+	common.Infof(ctx, "Success")
+}
+
+func BuildListsSanitree(
+	ctx context.Context,
+	ac api.Client,
+	siteID string,
+	restoreContainerPrefix, exportFolderName string,
+) *common.Sanitree[models.Siteable, models.Listable] {
+	common.Infof(ctx, "building sanitree for lists of site: %s", siteID)
+
+	site, err := ac.Sites().GetByID(ctx, siteID, api.CallConfig{})
+	if err != nil {
+		common.Fatal(
+			ctx,
+			fmt.Sprintf("finding site by id %q", siteID),
+			err)
+	}
+
+	cfg := api.CallConfig{
+		Select: idAnd("displayName", "list", "lastModifiedDateTime"),
+	}
+
+	lists, err := ac.Lists().GetLists(ctx, siteID, cfg)
+	if err != nil {
+		common.Fatal(
+			ctx,
+			fmt.Sprintf("finding lists of site with id %q", siteID),
+			err)
+	}
+
+	lists = filterToSupportedLists(lists)
+
+	filteredLists := filterListsByPrefix(lists, restoreContainerPrefix)
+
+	rootTreeName := ptr.Val(site.GetDisplayName())
+	// lists get stored into the local dir at destination/Lists/
+	if len(exportFolderName) > 0 {
+		rootTreeName = exportFolderName
+	}
+
+	root := &common.Sanitree[models.Siteable, models.Listable]{
+		Self:        site,
+		ID:          ptr.Val(site.GetId()),
+		Name:        rootTreeName,
+		CountLeaves: len(filteredLists),
+		Leaves:      map[string]*common.Sanileaf[models.Siteable, models.Listable]{},
+	}
+
+	for _, list := range filteredLists {
+		listID := ptr.Val(list.GetId())
+
+		listItems, err := ac.Lists().GetListItems(ctx, siteID, listID, api.CallConfig{})
+		if err != nil {
+			common.Fatal(
+				ctx,
+				fmt.Sprintf("finding listItems of list with id %q", listID),
+				err)
+		}
+
+		m := &common.Sanileaf[models.Siteable, models.Listable]{
+			Parent: root,
+			Self:   list,
+			ID:     listID,
+			Name:   ptr.Val(list.GetDisplayName()),
+			// using list item count as size for lists
+			Size: int64(len(listItems)),
+		}
+
+		root.Leaves[m.ID] = m
+	}
+
+	return root
+}
+
+func filterToSupportedLists(lists []models.Listable) []models.Listable {
+	filteredLists := make([]models.Listable, 0)
+
+	for _, list := range lists {
+		if !api.SkipListTemplates.HasKey(ptr.Val(list.GetList().GetTemplate())) {
+			filteredLists = append(filteredLists, list)
+		}
+	}
+
+	return filteredLists
+}
+
+func filterListsByPrefix(lists []models.Listable, prefix string) []models.Listable {
+	result := []models.Listable{}
+
+	for _, list := range lists {
+		for _, pfx := range strings.Split(prefix, ",") {
+			if strings.HasPrefix(ptr.Val(list.GetDisplayName()), pfx) {
+				result = append(result, list)
+				break
+			}
+		}
+	}
+
+	return result
+}
+
+func idAnd(ss ...string) []string {
+	id := []string{"id"}
+
+	if len(ss) == 0 {
+		return id
+	}
+
+	return append(id, ss...)
 }