add sharepoint to sanity tests (#3204)

Add sharepoint sanity testing.

---

#### Does this PR need a docs update or release note?

- [x] 🕐 Yes, but in a later PR

#### Type of change

- [x] 🌻 Feature

#### Issue(s)

* #3135

#### Test Plan

- [x] 💪 Manual
- [x] ⚡ Unit test
- [x] 💚 E2E

.github/workflows/sanity-test.yaml

@@ -52,13 +52,12 @@ jobs:
 
       - run: make build
 
-      - run: go build -o sanityCheck ./cmd/sanity_test
+      - run: go build -o sanityTest ./cmd/sanity_test
 
       - run: mkdir ${TEST_RESULT}
 
       - run: mkdir ${CORSO_LOG_DIR}
 
-      # run the tests
       - name: Version Test
         run: |  
           set -euo pipefail
@@ -90,7 +89,6 @@ jobs:
           
           echo result="$prefix" >> $GITHUB_OUTPUT 
 
-      # run the tests
       - name: Repo connect test
         run: |
           set -euo pipefail
@@ -107,6 +105,10 @@ jobs:
             exit 1
           fi
 
+##########################################################################################################################################
+
+# Exchange
+
       # generate new entries to roll into the next load test
       # only runs if the test was successful
       - name: New Data Creation
@@ -122,7 +124,6 @@ jobs:
           --destination Corso_Restore_st_${{ steps.repo-init.outputs.result }} \
           --count 4
 
-      # run the tests
       - name: Backup exchange test
         id: exchange-test
         run: |
@@ -145,7 +146,7 @@ jobs:
           data=$( echo $resultjson | jq -r '.[0] | .id' )
           echo result=$data >> $GITHUB_OUTPUT
 
-      # list all exchange backups
+      # list all backups
       - name: Backup exchange list test
         run: |
           set -euo pipefail
@@ -161,7 +162,7 @@ jobs:
             exit 1
           fi
 
-      # list the previous exchange backups
+      # list the previous backups
       - name: Backup exchange list single backup test
         run: |
           set -euo pipefail
@@ -178,7 +179,7 @@ jobs:
             exit 1
           fi
 
-      # test exchange restore
+      # restore
       - name: Backup exchange restore
         id: exchange-restore-test
         run: |
@@ -199,9 +200,9 @@ jobs:
           TEST_DATA: Corso_Restore_st_${{ steps.repo-init.outputs.result }}
         run: |
           set -euo pipefail
-          ./sanityCheck
+          ./sanityTest
 
-      # test incremental backup exchange
+      # incremental backup
       - name: Backup exchange incremental
         id: exchange-incremental-test
         run: |
@@ -223,7 +224,7 @@ jobs:
 
           echo result=$( echo $resultjson | jq -r '.[0] | .id' ) >> $GITHUB_OUTPUT
 
-      # test exchange restore
+      # restore from incremental
       - name: Backup incremantal exchange restore
         id: exchange-incremantal-restore-test
         run: |
@@ -245,11 +246,13 @@ jobs:
           BASE_BACKUP: ${{ steps.exchange-restore-test.outputs.result }}
         run: |
           set -euo pipefail
-          ./sanityCheck
+          ./sanityTest
 
-# Onedrive test
+##########################################################################################################################################
 
-       # generate new entries for OneDrive sanity test
+# Onedrive
+
+      # generate new entries for test
       - name: New Data Creation for OneDrive
         id: new-data-creation-onedrive
         working-directory: ./src/cmd/factory
@@ -269,7 +272,6 @@ jobs:
 
           echo result="$suffix" >> $GITHUB_OUTPUT
 
-      # run the tests
       - name: Backup onedrive test
         id: onedrive-test
         run: |
@@ -292,7 +294,7 @@ jobs:
           data=$( echo $resultjson | jq -r '.[0] | .id' )
           echo result=$data >> $GITHUB_OUTPUT
 
-      # list all onedrive backups
+      # list all backups
       - name: Backup onedrive list test
         run: |
           set -euo pipefail
@@ -308,8 +310,8 @@ jobs:
             exit 1
           fi
 
-      # list the previous onedrive backup
+      # list the previous backup
-      - name: Backup onedrive list one backup test
+      - name: Backup onedrive list test
         run: |
           set -euo pipefail
           echo -e "\nBackup OneDrive list one backup test\n" >> ${CORSO_LOG_FILE}
@@ -325,7 +327,7 @@ jobs:
             exit 1
           fi
 
-      # test onedrive restore
+      # restore
       - name: Backup onedrive restore
         id: onedrive-restore-test
         run: |
@@ -347,9 +349,9 @@ jobs:
           TEST_DATA: Corso_Restore_st_${{ steps.new-data-creation-onedrive.outputs.result }}
         run: |
           set -euo pipefail
-          ./sanityCheck
+          ./sanityTest
 
-       # generate some more enteries for incremental check
+      # generate some more enteries for incremental check
       - name: New Data Creation for Incremental OneDrive
         working-directory: ./src/cmd/factory
         env:
@@ -364,7 +366,7 @@ jobs:
           --destination Corso_Restore_st_${{ steps.new-data-creation-onedrive.outputs.result }} \
           --count 4
 
-      # test onedrive incremental
+      # incremental backup
       - name: Backup onedrive incremental
         id: onedrive-incremental-test
         run: |
@@ -387,7 +389,7 @@ jobs:
           data=$( echo $resultjson | jq -r '.[0] | .id' )
           echo result=$data >> $GITHUB_OUTPUT
           
-      # test onedrive restore
+      # restore from incremental
       - name: Backup onedrive restore
         id: onedrive-incremental-restore-test
         run: |
@@ -409,7 +411,149 @@ jobs:
           TEST_DATA: Corso_Restore_st_${{ steps.new-data-creation-onedrive.outputs.result }}
         run: |
           set -euo pipefail
-          ./sanityCheck
+          ./sanityTest
+
+##########################################################################################################################################
+
+# Sharepoint test
+
+      # TODO(keepers): generate new entries for test
+
+      - name: Backup sharepoint test
+        id: sharepoint-test
+        run: |
+          set -euo pipefail
+        echo -e "\nBackup SharePoint test\n" >> ${CORSO_LOG_FILE}
+
+          ./corso backup create sharepoint \
+          --no-stats \
+          --hide-progress \
+          --site "${CORSO_M365_TEST_SITE_URL}" \
+          --json \
+          2>&1 | tee $TEST_RESULT/backup_sharepoint.txt 
+
+          resultjson=$(sed -e '1,/Completed Backups/d' $TEST_RESULT/backup_sharepoint.txt ) 
+
+          if  [[ $( echo $resultjson | jq -r '.[0] | .errorCount') -ne 0 ]]; then
+            echo "backup was not successful"
+            exit 1
+          fi
+
+          data=$( echo $resultjson | jq -r '.[0] | .id' )
+          echo result=$data >> $GITHUB_OUTPUT
+
+      # list all backups
+      - name: Backup sharepoint list test
+        run: |
+          set -euo pipefail
+        echo -e "\nBackup List SharePoint test\n" >> ${CORSO_LOG_FILE}
+
+          ./corso backup list sharepoint \
+          --no-stats \
+          --hide-progress \
+          2>&1 | tee $TEST_RESULT/backup_sharepoint_list.txt 
+
+          if ! grep -q  ${{ steps.sharepoint-test.outputs.result }} $TEST_RESULT/backup_sharepoint_list.txt 
+          then
+            echo "listing of backup was not successful"
+            exit 1
+          fi
+
+      # list the previous backup
+      - name: Backup sharepoint list single backup test
+        run: |
+          set -euo pipefail
+        echo -e "\nBackup List single backup SharePoint test\n" >> ${CORSO_LOG_FILE}
+
+          ./corso backup list sharepoint \
+          --no-stats \
+          --hide-progress \
+          --backup "${{ steps.sharepoint-test.outputs.result }}" \
+          2>&1 | tee $TEST_RESULT/backup_sharepoint_list_single.txt
+
+          if ! grep -q  ${{ steps.sharepoint-test.outputs.result }} $TEST_RESULT/backup_sharepoint_list.txt 
+          then
+            echo "listing of backup was not successful"
+            exit 1
+          fi
+
+      # restore
+      - name: Backup sharepoint restore
+        id: sharepoint-restore-test
+        run: |
+          set -euo pipefail
+        echo -e "\nRestore SharePoint test\n" >> ${CORSO_LOG_FILE}
+
+          ./corso restore sharepoint \
+          --no-stats \
+          --hide-progress \
+          --backup "${{ steps.sharepoint-test.outputs.result }}" \
+          2>&1 | tee $TEST_RESULT/sharepoint-restore-test.txt 
+          echo result=$(grep -i -e 'Restoring to folder ' $TEST_RESULT/sharepoint-restore-test.txt | sed "s/Restoring to folder//")  >> $GITHUB_OUTPUT
+
+    # TODO: Add when supported      
+    # --restore-permissions \
+
+      - name: Restoration sharepoint check
+        env:
+          SANITY_RESTORE_FOLDER: ${{ steps.sharepoint-restore-test.outputs.result }}
+          SANITY_RESTORE_SERVICE: "sharepoint"
+        run: |
+          set -euo pipefail
+          ./sanityTest
+
+      # TODO(rkeepers): generate some more entries for incremental check
+
+      # incremental backup
+      - name: Backup sharepoint incremental
+        id: sharepoint-incremental-test
+        run: |
+          set -euo pipefail
+        echo -e "\nIncremental Backup SharePoint test\n" >> ${CORSO_LOG_FILE}
+
+          ./corso backup create sharepoint \
+          --no-stats \
+          --hide-progress \
+          --site "${CORSO_M365_TEST_SITE_URL}" \
+          --json \
+          2>&1 | tee $TEST_RESULT/backup_sharepoint_incremental.txt 
+          
+          resultjson=$(sed -e '1,/Completed Backups/d' $TEST_RESULT/backup_sharepoint_incremental.txt ) 
+
+          if  [[ $( echo $resultjson | jq -r '.[0] | .errorCount') -ne 0 ]]; then
+            echo "backup was not successful"
+            exit 1
+          fi
+
+          data=$( echo $resultjson | jq -r '.[0] | .id' )
+          echo result=$data >> $GITHUB_OUTPUT
+
+      # restore from incremental
+      - name: Backup sharepoint restore
+        id: sharepoint-incremental-restore-test
+        run: |
+          set -euo pipefail
+        echo -e "\nIncremental Restore SharePoint test\n" >> ${CORSO_LOG_FILE}
+
+          ./corso restore sharepoint \
+          --no-stats \
+          --hide-progress \
+          --backup "${{ steps.sharepoint-incremental-test.outputs.result }}" \
+          2>&1 | tee $TEST_RESULT/sharepoint-incremental-restore-test.txt 
+          echo result=$(grep -i -e 'Restoring to folder ' $TEST_RESULT/sharepoint-incremental-restore-test.txt | sed "s/Restoring to folder//")  >> $GITHUB_OUTPUT
+
+    # TODO: Add when supported      
+    # --restore-permissions \
+
+      - name: Restoration sharepoint check
+        env:
+          SANITY_RESTORE_FOLDER: ${{ steps.sharepoint-incremental-restore-test.outputs.result }}
+          SANITY_RESTORE_SERVICE: "sharepoint"
+        run: |
+          set -euo pipefail
+          ./sanityTest
+
+##########################################################################################################################################
 
       # Upload the original go test output as an artifact for later review.
       - name: Upload test log
@@ -421,7 +565,6 @@ jobs:
           if-no-files-found: error
           retention-days: 14
 
-      # run the tests
       - name: SHA info
         id: sha-info
         if: failure()
@@ -431,7 +574,6 @@ jobs:
           echo RUN_URL=${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}  >> $GITHUB_OUTPUT
           echo COMMIT_URL=${{ github.server_url }}/${{ github.repository }}/commit/${GITHUB_SHA} >> $GITHUB_OUTPUT
       
-      
       - name: Send Github Action failure to Slack
         id: slack-notification
         if: failure()

src/cmd/sanity_test/sanity_tests.go

@@ -63,6 +63,7 @@ func main() {
 	var (
 		client           = msgraphsdk.NewGraphServiceClient(adapter)
 		testUser         = tester.GetM365UserID(ctx)
+		testSite         = tester.GetM365SiteID(ctx)
 		testService      = os.Getenv("SANITY_RESTORE_SERVICE")
 		folder           = strings.TrimSpace(os.Getenv("SANITY_RESTORE_FOLDER"))
 		startTime, _     = mustGetTimeFromName(ctx, folder)
@@ -83,7 +84,9 @@ func main() {
 	case "exchange":
 		checkEmailRestoration(ctx, client, testUser, folder, dataFolder, baseBackupFolder, startTime)
 	case "onedrive":
-		checkOnedriveRestoration(ctx, client, testUser, folder, dataFolder, startTime)
+		checkOneDriveRestoration(ctx, client, testUser, folder, dataFolder, startTime)
+	case "sharepoint":
+		checkSharePointRestoration(ctx, client, testSite, folder, dataFolder, startTime)
 	default:
 		fatal(ctx, "no service specified", nil)
 	}
@@ -292,36 +295,88 @@ func checkAllSubFolder(
 // oneDrive
 // ---------------------------------------------------------------------------
 
-func checkOnedriveRestoration(
+func checkOneDriveRestoration(
 	ctx context.Context,
 	client *msgraphsdk.GraphServiceClient,
-	testUser,
+	userID, folderName, dataFolder string,
-	folderName, dataFolder string,
 	startTime time.Time,
 ) {
-	var (
-		// map itemID -> item size
-		fileSizes = make(map[string]int64)
-		// map itemID -> permission id -> []permission roles
-		folderPermission        = make(map[string][]permissionInfo)
-		restoreFile             = make(map[string]int64)
-		restoreFolderPermission = make(map[string][]permissionInfo)
-	)
-
 	drive, err := client.
-		UsersById(testUser).
+		UsersById(userID).
 		Drive().
 		Get(ctx, nil)
 	if err != nil {
 		fatal(ctx, "getting the drive:", err)
 	}
 
+	checkDriveRestoration(
+		ctx,
+		client,
+		userID,
+		folderName,
+		ptr.Val(drive.GetId()),
+		ptr.Val(drive.GetName()),
+		dataFolder,
+		startTime,
+		false)
+}
+
+// ---------------------------------------------------------------------------
+// sharePoint
+// ---------------------------------------------------------------------------
+
+func checkSharePointRestoration(
+	ctx context.Context,
+	client *msgraphsdk.GraphServiceClient,
+	siteID, folderName, dataFolder string,
+	startTime time.Time,
+) {
+	drive, err := client.
+		SitesById(siteID).
+		Drive().
+		Get(ctx, nil)
+	if err != nil {
+		fatal(ctx, "getting the drive:", err)
+	}
+
+	checkDriveRestoration(
+		ctx,
+		client,
+		siteID,
+		folderName,
+		ptr.Val(drive.GetId()),
+		ptr.Val(drive.GetName()),
+		dataFolder,
+		startTime,
+		true)
+}
+
+// ---------------------------------------------------------------------------
+// shared drive tests
+// ---------------------------------------------------------------------------
+
+func checkDriveRestoration(
+	ctx context.Context,
+	client *msgraphsdk.GraphServiceClient,
+	resourceOwner,
+	folderName,
+	driveID,
+	driveName,
+	dataFolder string,
+	startTime time.Time,
+	skipPermissionTest bool,
+) {
 	var (
-		driveID         = ptr.Val(drive.GetId())
+		// map itemID -> item size
-		driveName       = ptr.Val(drive.GetName())
+		fileSizes = make(map[string]int64)
-		restoreFolderID string
+		// map itemID -> permission id -> []permission roles
+		folderPermissions         = make(map[string][]permissionInfo)
+		restoreFile               = make(map[string]int64)
+		restoredFolderPermissions = make(map[string][]permissionInfo)
 	)
 
+	var restoreFolderID string
+
 	ctx = clues.Add(ctx, "drive_id", driveID, "drive_name", driveName)
 
 	response, err := client.
@@ -345,9 +400,7 @@ func checkOnedriveRestoration(
 		}
 
 		if itemName != dataFolder {
-			logger.Ctx(ctx).Infof("test data for %v folder: ", dataFolder)
+			logAndPrint(ctx, "test data for folder: %s", dataFolder)
-			fmt.Printf("test data for %v folder: ", dataFolder)
-
 			continue
 		}
 
@@ -363,27 +416,55 @@ func checkOnedriveRestoration(
 		// currently we don't restore blank folders.
 		// skip permission check for empty folders
 		if ptr.Val(driveItem.GetFolder().GetChildCount()) == 0 {
-			logger.Ctx(ctx).Info("skipped empty folder: ", itemName)
+			logAndPrint(ctx, "skipped empty folder: %s", itemName)
-			fmt.Println("skipped empty folder: ", itemName)
-
 			continue
 		}
 
-		folderPermission[itemName] = permissionIn(ctx, client, driveID, itemID)
+		folderPermissions[itemName] = permissionIn(ctx, client, driveID, itemID)
-		getOneDriveChildFolder(ctx, client, driveID, itemID, itemName, fileSizes, folderPermission, startTime)
+		getOneDriveChildFolder(ctx, client, driveID, itemID, itemName, fileSizes, folderPermissions, startTime)
 	}
 
-	getRestoredDrive(ctx, client, *drive.GetId(), restoreFolderID, restoreFile, restoreFolderPermission, startTime)
+	getRestoredDrive(ctx, client, driveID, restoreFolderID, restoreFile, restoredFolderPermissions, startTime)
 
-	for folderName, permissions := range folderPermission {
+	checkRestoredDriveItemPermissions(
+		ctx,
+		skipPermissionTest,
+		folderPermissions,
+		restoredFolderPermissions)
+
+	for fileName, expected := range fileSizes {
+		logAndPrint(ctx, "checking for file: %s", fileName)
+
+		got := restoreFile[fileName]
+
+		assert(
+			ctx,
+			func() bool { return expected == got },
+			fmt.Sprintf("different file size: %s", fileName),
+			expected,
+			got)
+	}
+
+	fmt.Println("Success")
+}
+
+func checkRestoredDriveItemPermissions(
+	ctx context.Context,
+	skip bool,
+	folderPermissions map[string][]permissionInfo,
+	restoredFolderPermissions map[string][]permissionInfo,
+) {
+	if skip {
+		return
+	}
+
+	for folderName, permissions := range folderPermissions {
 		logAndPrint(ctx, "checking for folder: %s", folderName)
 
-		restoreFolderPerm := restoreFolderPermission[folderName]
+		restoreFolderPerm := restoredFolderPermissions[folderName]
 
 		if len(permissions) < 1 {
-			logger.Ctx(ctx).Info("no permissions found in:", folderName)
+			logAndPrint(ctx, "no permissions found in: %s", folderName)
-			fmt.Println("no permissions found in:", folderName)
-
 			continue
 		}
 
@@ -413,22 +494,6 @@ func checkOnedriveRestoration(
 				restored.roles)
 		}
 	}
-
-	for fileName, expected := range fileSizes {
-		logger.Ctx(ctx).Info("checking for file: ", fileName)
-		fmt.Printf("checking for file: %s\n", fileName)
-
-		got := restoreFile[fileName]
-
-		assert(
-			ctx,
-			func() bool { return expected == got },
-			fmt.Sprintf("different file size: %s", fileName),
-			expected,
-			got)
-	}
-
-	fmt.Println("Success")
 }
 
 func getOneDriveChildFolder(

src/internal/tester/resource_owners.go

@@ -184,3 +184,16 @@ func M365SiteURL(t *testing.T) string {
 
 	return strings.ToLower(cfg[TestCfgSiteURL])
 }
+
+// GetM365SiteID returns a siteID string representing the m365SitteID described
+// by either the env var CORSO_M365_TEST_SITE_ID, the corso_test.toml config
+// file or the default value (in that order of priority).  The default is a
+// last-attempt fallback that will only work on alcion's testing org.
+func GetM365SiteID(ctx context.Context) string {
+	cfg, err := readTestConfig()
+	if err != nil {
+		logger.Ctx(ctx).Error(err, "retrieving m365 user id from test configuration")
+	}
+
+	return strings.ToLower(cfg[TestCfgSiteID])
+}