Release Sanity test (#2622)

## Description

Github workflow - Release sanity test which can run pre and post
release.
Testing- 
- version command
- repo init and connect
- backup and restore of exchange and onedrive
-  verify correct no. of emails restored
- verify file sizes restored and roles on folder are correct.
- incremental backup and verification of the same

**Further enhancement-**
- check actual email data instead of just no. of emails
- verify contacts and calendar
- permission in onedrive
- slack notification on failure 
- if we recieve an email in between a backup and restore, we will fail
the test as the counts won't match

**issues for handling enhancement-** 
https://github.com/alcionai/corso/issues/2743
https://github.com/alcionai/corso/issues/2742

## Does this PR need a docs update or release note?
- [ ] ⛔ No 

## Type of change

<!--- Please check the type of change your PR introduces: --->
- [x] 🤖 Test

## Issue(s)

<!-- Can reference multiple issues. Use one of the following "magic
words" - "closes, fixes" to auto-close the Github issue. -->
* https://github.com/alcionai/corso/pull/2622/

## Test Plan
- [ ] 💪 Manual

.github/workflows/sanity-test.yaml

@@ -0,0 +1,263 @@
+name: Sanity Testing
+on: 
+  push:
+    branches:
+      - main
+  workflow_dispatch:
+
+permissions:
+  # required to retrieve AWS credentials
+  id-token: write
+  contents: write
+
+# cancel currently running jobs if a new version of the branch is pushed
+concurrency:
+  group: sanity_testing-${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+
+jobs:
+  Sanity-Tests:
+    environment: Testing
+    runs-on: ubuntu-latest
+    env:
+      AZURE_CLIENT_ID: ${{ secrets.CLIENT_ID }}
+      AZURE_CLIENT_SECRET: ${{ secrets.CLIENT_SECRET }}
+      AZURE_TENANT_ID: ${{ secrets.TENANT_ID }}
+      CORSO_M365_TEST_USER_ID: ${{ secrets.CORSO_M365_TEST_USER_ID }}
+      CORSO_PASSPHRASE: ${{ secrets.INTEGRATION_TEST_CORSO_PASSPHRASE }}
+      TEST_RESULT: "test_results"
+      CORSO_BUCKET: ${{ secrets.CI_TESTS_S3_BUCKET }}
+
+    defaults:
+      run:
+        working-directory: src
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Setup Golang with cache
+        uses: magnetikonline/action-golang-cache@v3
+        with:
+          go-version-file: src/go.mod
+
+      - run: make build
+
+      - run: go build -o sanityCheck ./cmd/sanity_test
+
+      - run: mkdir test_results
+
+      # AWS creds
+      - name: Configure AWS credentials from Test account
+        uses: aws-actions/configure-aws-credentials@v1
+        with:
+          role-to-assume: ${{ secrets.AWS_IAM_ROLE }}
+          role-session-name: integration-testing
+          aws-region: us-east-1
+
+      # run the tests
+      - name: Version Test
+        run: |
+          set -euo pipefail
+          if [ $( ./corso --version | grep 'Corso version:' | wc -l) -ne 1 ]
+          then
+              echo "valid version not found"
+              exit 1
+          fi
+
+      - name: Repo init test
+        id: repo-init
+        env:
+          TEST_RESULT: "test_results"
+        run: |
+          set -euo pipefail 
+          prefix=`date +"%Y-%m-%d-%T"`
+
+          ./corso repo init s3 \
+          --hide-progress \
+          --prefix $prefix \
+          --bucket  ${CORSO_BUCKET}  2>&1 | tee $TEST_RESULT/initrepo.txt 
+
+          if ! grep -q 'Initialized a S3 repository within bucket' $TEST_RESULT/initrepo.txt 
+          then
+              echo "repo could not be initiated"
+              exit 1
+          fi
+          
+          echo result="$prefix" >> $GITHUB_OUTPUT 
+
+      # run the tests
+      - name: Repo connect test
+        run: |
+          set -euo pipefail
+          ./corso repo connect s3 \
+          --hide-progress \
+          --prefix ${{ steps.repo-init.outputs.result }} \
+          --bucket  ${CORSO_BUCKET} 2>&1 | tee $TEST_RESULT/connect.txt 
+
+          if ! grep -q 'Connected to S3 bucket' $TEST_RESULT/connect.txt 
+          then
+              echo "repo could not be connected"
+              exit 1
+          fi
+
+      # run the tests
+      - name: Backup exchange test
+        id: exchange-test
+        run: |
+          ./corso backup create exchange \
+          --user "${CORSO_M365_TEST_USER_ID}" \
+           --hide-progress 2>&1 | tee $TEST_RESULT/backup_exchange.txt 
+
+          if ! grep -q 'Completed (0 errors)' $TEST_RESULT/backup_exchange.txt
+          then
+              echo "backup was not successfull"
+              exit 1
+          fi
+          echo result=$(grep -i -e 'Completed (0 errors)' $TEST_RESULT/backup_exchange.txt  | awk '{print $2}') >> $GITHUB_OUTPUT
+
+      # list the backup exhange
+      - name: Backup exchange list test
+        run: |
+          set -euo pipefail
+          ./corso backup list exchange \
+          --hide-progress 2>&1 | tee $TEST_RESULT/backup_exchange_list.txt
+
+          if ! grep -q  ${{ steps.exchange-test.outputs.result }} $TEST_RESULT/backup_exchange_list.txt 
+          then
+              echo "listing of backup was not successfull"
+              exit 1
+          fi
+
+      # test exchange restore
+      - name: Backup exchange restore
+        id: exchange-restore-test
+        run: |
+          set -euo pipefail
+          ./corso restore exchange \
+          --hide-progress \
+          --backup "${{ steps.exchange-test.outputs.result }}" 2>&1 | tee $TEST_RESULT/exchange-restore-test.txt
+          echo result=$(grep -i -e 'Restoring to folder ' $TEST_RESULT/exchange-restore-test.txt | sed "s/Restoring to folder//" )  >> $GITHUB_OUTPUT
+        
+      - name: Restoration check
+        env:
+          RESTORE_FOLDER: ${{ steps.exchange-restore-test.outputs.result }}
+          RESTORE_SERVICE: "exchange"
+        run: |
+          set -euo pipefail
+          ./sanityCheck
+
+      # test incremental backup exhange
+      - name: Backup exchange incremental
+        id: exchange-incremental-test
+        run: |
+          set -euo pipefail
+          ./corso backup create exchange \
+          --user "${CORSO_M365_TEST_USER_ID}" \
+          --hide-progress 2>&1 | tee $TEST_RESULT/backup_exchange_incremental.txt 
+
+          if ! grep -q  'Completed (0 errors)' $TEST_RESULT/backup_exchange_incremental.txt 
+          then
+              echo "backup was not successful"
+              exit 1
+          fi
+
+          echo result=$(grep -i -e 'Completed (0 errors)' $TEST_RESULT/backup_exchange_incremental.txt  | awk '{print $2}') >> $GITHUB_OUTPUT
+
+      # test exchange restore
+      - name: Backup incremantal exchange restore
+        id: exchange-incremantal-restore-test
+        run: |
+          set -euo pipefail
+          ./corso restore exchange \
+          --hide-progress \
+          --backup "${{ steps.exchange-incremental-test.outputs.result }}" 2>&1 | tee $TEST_RESULT/exchange-incremantal-restore-test.txt
+          echo result=$(grep -i -e 'Restoring to folder ' $TEST_RESULT/exchange-incremantal-restore-test.txt | sed "s/Restoring to folder//" )  >> $GITHUB_OUTPUT
+        
+      - name: Restoration check
+        env:
+          RESTORE_FOLDER: ${{ steps.exchange-incremantal-restore-test.outputs.result }}
+          RESTORE_SERVICE: "exchange"
+        run: |
+          set -euo pipefail
+          ./sanityCheck
+
+        
+ # Onedrive test
+
+      # run the tests
+      - name: Backup onedrive test
+        id: onedrive-test
+        run: |
+          set -euo pipefail
+          ./corso backup create onedrive \
+          --user "${CORSO_M365_TEST_USER_ID}" \
+          --hide-progress 2>&1 | tee $TEST_RESULT/backup_onedrive.txt 
+
+          if ! grep -q 'Completed (0 errors)' $TEST_RESULT/backup_onedrive.txt
+          then
+              echo "backup was not successfull"
+              exit 1
+          fi
+
+          echo result=$(grep 'Completed (0 errors)' $TEST_RESULT/backup_onedrive.txt  | awk '{print $2}') >> $GITHUB_OUTPUT
+
+      # list the bakcup onedrive
+      - name: Backup onedrive list test
+        run: |
+          set -euo pipefail
+          ./corso backup list onedrive \
+          --hide-progress 2>&1 | tee $TEST_RESULT/backup_onedrive_list.txt 
+
+          if ! grep -q  ${{ steps.onedrive-test.outputs.result }} $TEST_RESULT/backup_onedrive_list.txt 
+          then
+              echo "listing of backup was not successfull"
+              exit 1
+          fi
+
+      # test onedrive restore
+      - name: Backup onedrive restore
+        id: onedrive-restore-test
+        run: |
+          set -euo pipefail
+          ./corso restore onedrive --backup "${{ steps.onedrive-test.outputs.result }}"  --hide-progress 2>&1 | tee $TEST_RESULT/onedrive-restore-test.txt 
+          echo result=$(grep -i -e 'Restoring to folder ' $TEST_RESULT/onedrive-restore-test.txt | sed "s/Restoring to folder//")  >> $GITHUB_OUTPUT
+
+      - name: Restoration oneDrive check
+        env:
+          RESTORE_FOLDER: ${{ steps.onedrive-restore-test.outputs.result }}
+        run: |
+          set -euo pipefail
+          ./sanityCheck
+
+      # test onedrive incremental
+      - name: Backup onedrive test
+        id: onedrive-incremental-test
+        run: |
+          set -euo pipefail
+          ./corso backup create onedrive \
+          --user "${CORSO_M365_TEST_USER_ID}"\
+           --hide-progress 2>&1 | tee $TEST_RESULT/backup_onedrive_incremental.txt 
+          
+          if ! grep -q 'Completed (0 errors)' $TEST_RESULT/backup_onedrive_incremental.txt
+          then
+              echo "backup was not successfull"
+              exit 1
+          fi
+
+          echo result=$(grep -i -e 'Completed (0 errors)' $TEST_RESULT/backup_onedrive_incremental.txt  | awk '{print $2}') >> $GITHUB_OUTPUT
+
+      # test onedrive restore
+      - name: Backup onedrive restore
+        id: onedrive-incremental-restore-test
+        run: |
+          set -euo pipefail
+          ./corso restore onedrive --backup "${{ steps.onedrive-incremental-test.outputs.result }}"  --hide-progress 2>&1 | tee $TEST_RESULT/onedrive-incremental-restore-test.txt 
+          echo result=$(grep -i -e 'Restoring to folder ' $TEST_RESULT/onedrive-incremental-restore-test.txt | sed "s/Restoring to folder//")  >> $GITHUB_OUTPUT
+
+      - name: Restoration oneDrive check
+        env:
+          RESTORE_FOLDER: ${{ steps.onedrive-incremental-restore-test.outputs.result }}
+        run: |
+          set -euo pipefail
+          ./sanityCheck
+      

src/cli/restore/exchange.go

@@ -217,6 +217,7 @@ func restoreExchangeCmd(cmd *cobra.Command, args []string) error {
 	defer utils.CloseRepo(ctx, r)
 
 	dest := control.DefaultRestoreDestination(common.SimpleDateTime)
+	Infof(ctx, "Restoring to folder %s", dest.ContainerName)
 
 	sel := utils.IncludeExchangeRestoreDataSelectors(opts)
 	utils.FilterExchangeRestoreInfoSelectors(sel, opts)

src/cli/restore/onedrive.go

@@ -160,6 +160,7 @@ func restoreOneDriveCmd(cmd *cobra.Command, args []string) error {
 	defer utils.CloseRepo(ctx, r)
 
 	dest := control.DefaultRestoreDestination(common.SimpleDateTimeOneDrive)
+	Infof(ctx, "Restoring to folder %s", dest.ContainerName)
 
 	sel := utils.IncludeOneDriveRestoreDataSelectors(opts)
 	utils.FilterOneDriveRestoreInfoSelectors(sel, opts)

src/cli/restore/sharepoint.go

@@ -173,6 +173,7 @@ func restoreSharePointCmd(cmd *cobra.Command, args []string) error {
 	defer utils.CloseRepo(ctx, r)
 
 	dest := control.DefaultRestoreDestination(common.SimpleDateTimeOneDrive)
+	Infof(ctx, "Restoring to folder %s", dest.ContainerName)
 
 	sel := utils.IncludeSharePointRestoreDataSelectors(opts)
 	utils.FilterSharePointRestoreInfoSelectors(sel, opts)

src/cmd/factory/impl/common.go

@@ -10,7 +10,7 @@ import (
 	"github.com/google/uuid"
 	"github.com/pkg/errors"
 
-	. "github.com/alcionai/corso/src/cli/print"
+	"github.com/alcionai/corso/src/cli/print"
 	"github.com/alcionai/corso/src/internal/common"
 	"github.com/alcionai/corso/src/internal/connector"
 	"github.com/alcionai/corso/src/internal/connector/graph"
@@ -83,6 +83,7 @@ func generateAndRestoreItems(
 	// TODO: fit the destination to the containers
 	dest := control.DefaultRestoreDestination(common.SimpleTimeTesting)
 	dest.ContainerName = destFldr
+	print.Infof(ctx, "Restoring to folder %s", dest.ContainerName)
 
 	dataColls, err := buildCollections(
 		service,
@@ -94,7 +95,7 @@ func generateAndRestoreItems(
 		return nil, err
 	}
 
-	Infof(ctx, "Generating %d %s items in %s\n", howMany, cat, Destination)
+	print.Infof(ctx, "Generating %d %s items in %s\n", howMany, cat, Destination)
 
 	return gc.RestoreDataCollections(ctx, version.Backup, acct, sel, dest, opts, dataColls, errs)
 }

src/cmd/sanity_test/sanity_tests.go

@@ -0,0 +1,231 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"reflect"
+	"strings"
+	"time"
+
+	"github.com/alcionai/corso/src/internal/common/ptr"
+	"github.com/alcionai/corso/src/internal/connector/graph"
+	msgraphsdk "github.com/microsoftgraph/msgraph-sdk-go"
+	"github.com/microsoftgraph/msgraph-sdk-go/models"
+	"github.com/microsoftgraph/msgraph-sdk-go/users"
+)
+
+func main() {
+	adapter, err := graph.CreateAdapter(
+		os.Getenv("AZURE_TENANT_ID"),
+		os.Getenv("AZURE_CLIENT_ID"),
+		os.Getenv("AZURE_CLIENT_SECRET"),
+	)
+	if err != nil {
+		fmt.Println("error while creating adapter: ", err)
+		os.Exit(1)
+
+		return
+	}
+
+	testUser := os.Getenv("CORSO_M365_TEST_USER_ID")
+	folder := strings.TrimSpace(os.Getenv("RESTORE_FOLDER"))
+
+	restoreStartTime := strings.SplitAfter(folder, "Corso_Restore_")[1]
+	startTime, _ := time.Parse(time.RFC822, restoreStartTime)
+
+	fmt.Println("Restore folder: ", folder)
+
+	client := msgraphsdk.NewGraphServiceClient(adapter)
+
+	switch service := os.Getenv("RESTORE_SERVICE"); service {
+	case "exchange":
+		checkEmailRestoration(client, testUser, folder, startTime)
+	default:
+		checkOnedriveRestoration(client, testUser, folder, startTime)
+	}
+}
+
+func checkEmailRestoration(client *msgraphsdk.GraphServiceClient, testUser, folderName string, startTime time.Time) {
+	var (
+		messageCount  = make(map[string]int32)
+		restoreFolder models.MailFolderable
+	)
+
+	builder := client.UsersById(testUser).MailFolders()
+
+	for {
+		result, err := builder.Get(context.Background(), nil)
+		if err != nil {
+			fmt.Printf("Error getting the drive: %v\n", err)
+			os.Exit(1)
+		}
+
+		res := result.GetValue()
+
+		for _, r := range res {
+			name := *r.GetDisplayName()
+
+			var rStartTime time.Time
+
+			restoreStartTime := strings.SplitAfter(name, "Corso_Restore_")
+			if len(restoreStartTime) > 1 {
+				rStartTime, _ = time.Parse(time.RFC822, restoreStartTime[1])
+				if startTime.Before(rStartTime) {
+					fmt.Printf("The restore folder %s was created after %s. Will skip check.", name, folderName)
+					continue
+				}
+			}
+
+			if name == folderName {
+				restoreFolder = r
+				continue
+			}
+
+			messageCount[*r.GetDisplayName()] = *r.GetTotalItemCount()
+		}
+
+		link, ok := ptr.ValOK(result.GetOdataNextLink())
+		if !ok {
+			break
+		}
+
+		builder = users.NewItemMailFoldersRequestBuilder(link, client.GetAdapter())
+	}
+
+	user := client.UsersById(testUser)
+	folder := user.MailFoldersById(*restoreFolder.GetId())
+
+	childFolder, err := folder.ChildFolders().Get(context.Background(), nil)
+	if err != nil {
+		fmt.Printf("Error getting the drive: %v\n", err)
+		os.Exit(1)
+	}
+
+	for _, restore := range childFolder.GetValue() {
+		if messageCount[*restore.GetDisplayName()] != *restore.GetTotalItemCount() {
+			fmt.Println("Restore was not succesfull for: ",
+				*restore.GetDisplayName(),
+				"Folder count: ",
+				messageCount[*restore.GetDisplayName()],
+				"Restore count: ",
+				*restore.GetTotalItemCount())
+			os.Exit(1)
+		}
+	}
+}
+
+func checkOnedriveRestoration(client *msgraphsdk.GraphServiceClient, testUser, folderName string, startTime time.Time) {
+	file := make(map[string]int64)
+	folderPermission := make(map[string][]string)
+	restoreFolderID := ""
+
+	drive, err := client.UsersById(testUser).Drive().Get(context.Background(), nil)
+	if err != nil {
+		fmt.Printf("Error getting the drive: %v\n", err)
+		os.Exit(1)
+	}
+
+	response, err := client.DrivesById(*drive.GetId()).Root().Children().Get(context.Background(), nil)
+	if err != nil {
+		fmt.Printf("Error getting drive by id: %v\n", err)
+		os.Exit(1)
+	}
+
+	for _, driveItem := range response.GetValue() {
+		if *driveItem.GetName() == folderName {
+			restoreFolderID = *driveItem.GetId()
+			continue
+		}
+
+		var rStartTime time.Time
+
+		restoreStartTime := strings.SplitAfter(*driveItem.GetName(), "Corso_Restore_")
+		if len(restoreStartTime) > 1 {
+			rStartTime, _ = time.Parse(time.RFC822, restoreStartTime[1])
+			if startTime.Before(rStartTime) {
+				fmt.Printf("The restore folder %s was created after %s. Will skip check.", *driveItem.GetName(), folderName)
+				continue
+			}
+		}
+
+		// if it's a file check the size
+		if driveItem.GetFile() != nil {
+			file[*driveItem.GetName()] = *driveItem.GetSize()
+		}
+
+		if driveItem.GetFolder() != nil {
+			permission, err := client.
+				DrivesById(*drive.GetId()).
+				ItemsById(*driveItem.GetId()).
+				Permissions().
+				Get(context.TODO(), nil)
+			if err != nil {
+				fmt.Printf("Error getting item by id: %v\n", err)
+				os.Exit(1)
+			}
+
+			// check if permission are correct on folder
+			for _, permission := range permission.GetValue() {
+				folderPermission[*driveItem.GetName()] = permission.GetRoles()
+			}
+
+			continue
+		}
+	}
+
+	checkFileData(client, *drive.GetId(), restoreFolderID, file, folderPermission)
+
+	fmt.Println("Success")
+}
+
+func checkFileData(
+	client *msgraphsdk.GraphServiceClient,
+	driveID,
+	restoreFolderID string,
+	file map[string]int64,
+	folderPermission map[string][]string,
+) {
+	itemBuilder := client.DrivesById(driveID).ItemsById(restoreFolderID)
+
+	restoreResponses, err := itemBuilder.Children().Get(context.Background(), nil)
+	if err != nil {
+		fmt.Printf("Error getting child folder: %v\n", err)
+		os.Exit(1)
+	}
+
+	for _, restoreData := range restoreResponses.GetValue() {
+		restoreName := *restoreData.GetName()
+
+		if restoreData.GetFile() != nil {
+			if *restoreData.GetSize() != file[restoreName] {
+				fmt.Printf("Size of file %s is different in drive %d and restored file: %d ",
+					restoreName,
+					file[restoreName],
+					*restoreData.GetSize())
+				os.Exit(1)
+			}
+
+			continue
+		}
+
+		itemBuilder := client.DrivesById(driveID).ItemsById(*restoreData.GetId())
+
+		permissionColl, err := itemBuilder.Permissions().Get(context.TODO(), nil)
+		if err != nil {
+			fmt.Printf("Error getting permission: %v\n", err)
+			os.Exit(1)
+		}
+
+		userPermission := []string{}
+
+		for _, perm := range permissionColl.GetValue() {
+			userPermission = perm.GetRoles()
+		}
+
+		if !reflect.DeepEqual(folderPermission[restoreName], userPermission) {
+			fmt.Printf("Permission mismatch for %s.", restoreName)
+			os.Exit(1)
+		}
+	}
+}

src/go.mod

@@ -71,7 +71,7 @@ require (
 	github.com/golang/protobuf v1.5.2 // indirect
 	github.com/hashicorp/errwrap v1.0.0 // indirect
 	github.com/hashicorp/golang-lru v0.5.4 // indirect
-	github.com/inconshreveable/mousetrap v1.0.1 // indirect
+	github.com/inconshreveable/mousetrap v1.1.0 // indirect
 	github.com/jmespath/go-jmespath v0.4.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
 	github.com/klauspost/compress v1.15.12 // indirect

src/go.sum

@@ -209,8 +209,9 @@ github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4=
 github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
 github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
 github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
-github.com/inconshreveable/mousetrap v1.0.1 h1:U3uMjPSQEBMNp1lFxmllqCPM6P5u/Xq7Pgzkat/bFNc=
 github.com/inconshreveable/mousetrap v1.0.1/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
+github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
+github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
 github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg=
 github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
 github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8=