From 07da7f16a127e260e67b9fbaf6c80050b45b1187 Mon Sep 17 00:00:00 2001 From: Hitesh Pattanayak <48874082+HiteshRepo@users.noreply.github.com> Date: Tue, 23 Jan 2024 17:38:16 +0530 Subject: [PATCH] adds local sanity tests for lists (#5031) adds local sanity tests for lists for reference: - `lists` do not have `folders` concept, they are directly created in a site. As a result cannot be restored to a different folder to test. Therefore they need to be created in adjacent to the 'backed-up' lists. Due to this there is a requirement of `SANITY_TEST_RESTORE_CONTAINER_PREFIX`. Using this, during restore check, this env-var is used to find restored lists. - Similarly, `SANITY_TEST_SOURCE_CONTAINER` is also used a prefix matcher to filter created test lists - When exported, lists are stored in local into the provided directory as `/Lists//.json` should go in after: - https://github.com/alcionai/corso/pull/5024 - https://github.com/alcionai/corso/pull/5026 #### Does this PR need a docs update or release note? - [x] :no_entry: No #### Type of change - [x] :robot: Supportability/Tests #### Issue(s) #4754 #### Test Plan - [x] :muscle: Manual - [x] :zap: Unit test - [x] :green_heart: E2E --- src/cmd/sanity_test/common/common.go | 37 +++-- src/cmd/sanity_test/common/filepath.go | 67 ++++++--- src/cmd/sanity_test/export/sharepoint.go | 141 ++++++++++++++++-- src/cmd/sanity_test/restore/sharepoint.go | 165 ++++++++++++++++++++-- 4 files changed, 355 insertions(+), 55 deletions(-) diff --git a/src/cmd/sanity_test/common/common.go b/src/cmd/sanity_test/common/common.go index 3008fa13a..85835d52f 100644 --- a/src/cmd/sanity_test/common/common.go +++ b/src/cmd/sanity_test/common/common.go @@ -19,33 +19,40 @@ type PermissionInfo struct { } const ( - sanityBackupID = "SANITY_BACKUP_ID" - sanityTestSourceContainer = "SANITY_TEST_SOURCE_CONTAINER" - sanityTestRestoreContainer = "SANITY_TEST_RESTORE_CONTAINER" - sanityTestUser = "SANITY_TEST_USER" + sanityBackupID = "SANITY_BACKUP_ID" + sanityTestSourceContainer = "SANITY_TEST_SOURCE_CONTAINER" + sanityTestRestoreContainer = "SANITY_TEST_RESTORE_CONTAINER" + sanityTestRestoreContainerPrefix = "SANITY_TEST_RESTORE_CONTAINER_PREFIX" + sanityTestUser = "SANITY_TEST_USER" + sanityTestCategory = "SANITY_TEST_CATEGORY" ) type Envs struct { BackupID string SourceContainer string RestoreContainer string - GroupID string - SiteID string - UserID string - TeamSiteID string + // applies for sharepoint lists only + RestoreContainerPrefix string + Category string + GroupID string + SiteID string + UserID string + TeamSiteID string } func EnvVars(ctx context.Context) Envs { folder := strings.TrimSpace(os.Getenv(sanityTestRestoreContainer)) e := Envs{ - BackupID: os.Getenv(sanityBackupID), - SourceContainer: os.Getenv(sanityTestSourceContainer), - RestoreContainer: folder, - GroupID: tconfig.GetM365TeamID(ctx), - SiteID: tconfig.GetM365SiteID(ctx), - UserID: tconfig.GetM365UserID(ctx), - TeamSiteID: tconfig.GetM365TeamSiteID(ctx), + BackupID: os.Getenv(sanityBackupID), + SourceContainer: os.Getenv(sanityTestSourceContainer), + RestoreContainer: folder, + Category: os.Getenv(sanityTestCategory), + RestoreContainerPrefix: os.Getenv(sanityTestRestoreContainerPrefix), + GroupID: tconfig.GetM365TeamID(ctx), + SiteID: tconfig.GetM365SiteID(ctx), + UserID: tconfig.GetM365UserID(ctx), + TeamSiteID: tconfig.GetM365TeamSiteID(ctx), } if len(os.Getenv(sanityTestUser)) > 0 { diff --git a/src/cmd/sanity_test/common/filepath.go b/src/cmd/sanity_test/common/filepath.go index 1b2c01a86..06857f7f1 100644 --- a/src/cmd/sanity_test/common/filepath.go +++ b/src/cmd/sanity_test/common/filepath.go @@ -20,31 +20,18 @@ func BuildFilepathSanitree( info os.FileInfo, err error, ) error { - if err != nil { - Fatal(ctx, "error passed to filepath walker", err) - } - - relPath, err := filepath.Rel(rootDir, p) - if err != nil { - Fatal(ctx, "getting relative filepath", err) - } - - if info != nil { - Debugf(ctx, "adding: %s", relPath) - } - if root == nil { - root = &Sanitree[fs.FileInfo, fs.FileInfo]{ - Self: info, - ID: info.Name(), - Name: info.Name(), - Leaves: map[string]*Sanileaf[fs.FileInfo, fs.FileInfo]{}, - Children: map[string]*Sanitree[fs.FileInfo, fs.FileInfo]{}, - } - + root = CreateNewRoot(info, true) return nil } + relPath := GetRelativePath( + ctx, + rootDir, + p, + info, + err) + elems := path.Split(relPath) node := root.NodeAt(ctx, elems[:len(elems)-1]) @@ -78,3 +65,41 @@ func BuildFilepathSanitree( return root } + +func CreateNewRoot(info fs.FileInfo, initChildren bool) *Sanitree[fs.FileInfo, fs.FileInfo] { + root := &Sanitree[fs.FileInfo, fs.FileInfo]{ + Self: info, + ID: info.Name(), + Name: info.Name(), + Leaves: map[string]*Sanileaf[fs.FileInfo, fs.FileInfo]{}, + Children: map[string]*Sanitree[fs.FileInfo, fs.FileInfo]{}, + } + + if initChildren { + root.Children = map[string]*Sanitree[fs.FileInfo, fs.FileInfo]{} + } + + return root +} + +func GetRelativePath( + ctx context.Context, + rootDir, p string, + info fs.FileInfo, + fileWalkerErr error, +) string { + if fileWalkerErr != nil { + Fatal(ctx, "error passed to filepath walker", fileWalkerErr) + } + + relPath, err := filepath.Rel(rootDir, p) + if err != nil { + Fatal(ctx, "getting relative filepath", err) + } + + if info != nil { + Debugf(ctx, "adding: %s", relPath) + } + + return relPath +} diff --git a/src/cmd/sanity_test/export/sharepoint.go b/src/cmd/sanity_test/export/sharepoint.go index eceea6ca5..faf120d49 100644 --- a/src/cmd/sanity_test/export/sharepoint.go +++ b/src/cmd/sanity_test/export/sharepoint.go @@ -2,10 +2,20 @@ package export import ( "context" + "io" + "io/fs" + "os" "path/filepath" + "strings" + + "github.com/alcionai/clues" + "github.com/microsoftgraph/msgraph-sdk-go/models" + "github.com/tidwall/gjson" "github.com/alcionai/corso/src/cmd/sanity_test/common" "github.com/alcionai/corso/src/cmd/sanity_test/driveish" + "github.com/alcionai/corso/src/cmd/sanity_test/restore" + "github.com/alcionai/corso/src/pkg/path" "github.com/alcionai/corso/src/pkg/services/m365/api" ) @@ -14,15 +24,128 @@ func CheckSharePointExport( ac api.Client, envs common.Envs, ) { - drive, err := ac.Sites().GetDefaultDrive(ctx, envs.SiteID) - if err != nil { - common.Fatal(ctx, "getting the drive:", err) + if envs.Category == path.ListsCategory.String() { + CheckSharepointListsExport(ctx, ac, envs) } - envs.RestoreContainer = filepath.Join(envs.RestoreContainer, "Libraries/Documents") // check in default loc - driveish.CheckExport( - ctx, - ac, - drive, - envs) + if envs.Category == path.LibrariesCategory.String() { + drive, err := ac.Sites().GetDefaultDrive(ctx, envs.SiteID) + if err != nil { + common.Fatal(ctx, "getting the drive:", err) + } + + envs.RestoreContainer = filepath.Join(envs.RestoreContainer, "Libraries/Documents") // check in default loc + driveish.CheckExport( + ctx, + ac, + drive, + envs) + } +} + +func CheckSharepointListsExport( + ctx context.Context, + ac api.Client, + envs common.Envs, +) { + exportFolderName := path.ListsCategory.HumanString() + + sourceTree := restore.BuildListsSanitree(ctx, ac, envs.SiteID, envs.SourceContainer, exportFolderName) + + listsExportDir := filepath.Join(envs.RestoreContainer, exportFolderName) + exportedTree := BuildFilepathSanitreeForSharepointLists(ctx, listsExportDir) + + ctx = clues.Add( + ctx, + "export_container_id", exportedTree.ID, + "export_container_name", exportedTree.Name, + "source_container_id", sourceTree.ID, + "source_container_name", sourceTree.Name) + + comparator := func( + ctx context.Context, + expect *common.Sanitree[models.Siteable, models.Listable], + result *common.Sanitree[fs.FileInfo, fs.FileInfo], + ) { + modifiedResultLeaves := map[string]*common.Sanileaf[fs.FileInfo, fs.FileInfo]{} + + for key, val := range result.Leaves { + fixedName := strings.TrimSuffix(key, ".json") + + modifiedResultLeaves[fixedName] = val + } + + common.CompareLeaves(ctx, expect.Leaves, modifiedResultLeaves, nil) + } + + common.CompareDiffTrees( + ctx, + sourceTree, + exportedTree, + comparator) + + common.Infof(ctx, "Success") +} + +func BuildFilepathSanitreeForSharepointLists( + ctx context.Context, + rootDir string, +) *common.Sanitree[fs.FileInfo, fs.FileInfo] { + var root *common.Sanitree[fs.FileInfo, fs.FileInfo] + + walker := func( + p string, + info os.FileInfo, + err error, + ) error { + if root == nil { + root = common.CreateNewRoot(info, false) + return nil + } + + relPath := common.GetRelativePath( + ctx, + rootDir, + p, + info, + err) + + if !info.IsDir() { + file, err := os.Open(p) + if err != nil { + common.Fatal(ctx, "opening file to read", err) + } + defer file.Close() + + content, err := io.ReadAll(file) + if err != nil { + common.Fatal(ctx, "reading file", err) + } + + res := gjson.Get(string(content), "items.#") + itemsCount := res.Num + + elems := path.Split(relPath) + + node := root.NodeAt(ctx, elems[:len(elems)-2]) + node.CountLeaves++ + node.Leaves[info.Name()] = &common.Sanileaf[fs.FileInfo, fs.FileInfo]{ + Parent: node, + Self: info, + ID: info.Name(), + Name: info.Name(), + // using list item count as size for lists + Size: int64(itemsCount), + } + } + + return nil + } + + err := filepath.Walk(rootDir, walker) + if err != nil { + common.Fatal(ctx, "walking filepath", err) + } + + return root } diff --git a/src/cmd/sanity_test/restore/sharepoint.go b/src/cmd/sanity_test/restore/sharepoint.go index 9fbb2a7f1..85094b8eb 100644 --- a/src/cmd/sanity_test/restore/sharepoint.go +++ b/src/cmd/sanity_test/restore/sharepoint.go @@ -2,9 +2,15 @@ package restore import ( "context" + "fmt" + "strings" + + "github.com/alcionai/clues" + "github.com/microsoftgraph/msgraph-sdk-go/models" "github.com/alcionai/corso/src/cmd/sanity_test/common" "github.com/alcionai/corso/src/cmd/sanity_test/driveish" + "github.com/alcionai/corso/src/internal/common/ptr" "github.com/alcionai/corso/src/pkg/services/m365/api" ) @@ -13,16 +19,155 @@ func CheckSharePointRestoration( ac api.Client, envs common.Envs, ) { - drive, err := ac.Sites().GetDefaultDrive(ctx, envs.SiteID) - if err != nil { - common.Fatal(ctx, "getting site's default drive:", err) + if envs.Category == "lists" { + CheckSharePointListsRestoration(ctx, ac, envs) } - driveish.CheckRestoration( - ctx, - ac, - drive, - envs, - // skip permissions tests - nil) + if envs.Category == "libraries" { + drive, err := ac.Sites().GetDefaultDrive(ctx, envs.SiteID) + if err != nil { + common.Fatal(ctx, "getting site's default drive:", err) + } + + driveish.CheckRestoration( + ctx, + ac, + drive, + envs, + // skip permissions tests + nil) + } +} + +func CheckSharePointListsRestoration( + ctx context.Context, + ac api.Client, + envs common.Envs, +) { + restoredTree := BuildListsSanitree(ctx, ac, envs.SiteID, envs.RestoreContainerPrefix, "") + sourceTree := BuildListsSanitree(ctx, ac, envs.SiteID, envs.SourceContainer, "") + + ctx = clues.Add( + ctx, + "restore_container_id", restoredTree.ID, + "restore_container_name", restoredTree.Name, + "source_container_id", sourceTree.ID, + "source_container_name", sourceTree.Name) + + common.CompareDiffTrees[models.Siteable, models.Listable]( + ctx, + sourceTree, + restoredTree, + nil) + + common.Infof(ctx, "Success") +} + +func BuildListsSanitree( + ctx context.Context, + ac api.Client, + siteID string, + restoreContainerPrefix, exportFolderName string, +) *common.Sanitree[models.Siteable, models.Listable] { + common.Infof(ctx, "building sanitree for lists of site: %s", siteID) + + site, err := ac.Sites().GetByID(ctx, siteID, api.CallConfig{}) + if err != nil { + common.Fatal( + ctx, + fmt.Sprintf("finding site by id %q", siteID), + err) + } + + cfg := api.CallConfig{ + Select: idAnd("displayName", "list", "lastModifiedDateTime"), + } + + lists, err := ac.Lists().GetLists(ctx, siteID, cfg) + if err != nil { + common.Fatal( + ctx, + fmt.Sprintf("finding lists of site with id %q", siteID), + err) + } + + lists = filterToSupportedLists(lists) + + filteredLists := filterListsByPrefix(lists, restoreContainerPrefix) + + rootTreeName := ptr.Val(site.GetDisplayName()) + // lists get stored into the local dir at destination/Lists/ + if len(exportFolderName) > 0 { + rootTreeName = exportFolderName + } + + root := &common.Sanitree[models.Siteable, models.Listable]{ + Self: site, + ID: ptr.Val(site.GetId()), + Name: rootTreeName, + CountLeaves: len(filteredLists), + Leaves: map[string]*common.Sanileaf[models.Siteable, models.Listable]{}, + } + + for _, list := range filteredLists { + listID := ptr.Val(list.GetId()) + + listItems, err := ac.Lists().GetListItems(ctx, siteID, listID, api.CallConfig{}) + if err != nil { + common.Fatal( + ctx, + fmt.Sprintf("finding listItems of list with id %q", listID), + err) + } + + m := &common.Sanileaf[models.Siteable, models.Listable]{ + Parent: root, + Self: list, + ID: listID, + Name: ptr.Val(list.GetDisplayName()), + // using list item count as size for lists + Size: int64(len(listItems)), + } + + root.Leaves[m.ID] = m + } + + return root +} + +func filterToSupportedLists(lists []models.Listable) []models.Listable { + filteredLists := make([]models.Listable, 0) + + for _, list := range lists { + if !api.SkipListTemplates.HasKey(ptr.Val(list.GetList().GetTemplate())) { + filteredLists = append(filteredLists, list) + } + } + + return filteredLists +} + +func filterListsByPrefix(lists []models.Listable, prefix string) []models.Listable { + result := []models.Listable{} + + for _, list := range lists { + for _, pfx := range strings.Split(prefix, ",") { + if strings.HasPrefix(ptr.Val(list.GetDisplayName()), pfx) { + result = append(result, list) + break + } + } + } + + return result +} + +func idAnd(ss ...string) []string { + id := []string{"id"} + + if len(ss) == 0 { + return id + } + + return append(id, ss...) }