diff --git a/.github/actions/backup-restore-test/action.yml b/.github/actions/backup-restore-test/action.yml index 3baf7c97c..2916bb34d 100644 --- a/.github/actions/backup-restore-test/action.yml +++ b/.github/actions/backup-restore-test/action.yml @@ -18,6 +18,10 @@ inputs: description: Arguments to pass for restore; restore is skipped when missing. required: false default: "" + export-args: + description: Arguments to pass for export. + required: false + default: "" restore-container: description: Folder to use for testing required: true @@ -32,6 +36,9 @@ inputs: description: Runs export tests when true required: false default: false + category: + description: category of data for given service + required: false outputs: backup-id: @@ -49,7 +56,9 @@ runs: echo Backup ${{ inputs.service }} ${{ inputs.kind }} echo "---------------------------" set -euo pipefail - CORSO_LOG_FILE=${{ inputs.log-dir }}/gotest-${{ inputs.service }}-backup-${{inputs.kind }}.log + CATEGORY_SUFFIX="" + [[ -n "${{ inputs.category }}" ]] && CATEGORY_SUFFIX="-${{ inputs.category }}" + CORSO_LOG_FILE=${{ inputs.log-dir }}/gotest-${{ inputs.service }}${CATEGORY_SUFFIX}-backup-${{inputs.kind }}.log ./corso backup create '${{ inputs.service }}' \ --no-stats --hide-progress --json \ ${{ inputs.backup-args }} | @@ -68,7 +77,9 @@ runs: echo Restore ${{ inputs.service }} ${{ inputs.kind }} echo "---------------------------" set -euo pipefail - CORSO_LOG_FILE=${{ inputs.log-dir }}/gotest-${{ inputs.service }}-restore-${{inputs.kind }}.log + CATEGORY_SUFFIX="" + [[ -n "${{ inputs.category }}" ]] && CATEGORY_SUFFIX="-${{ inputs.category }}" + CORSO_LOG_FILE=${{ inputs.log-dir }}/gotest-${{ inputs.service }}${CATEGORY_SUFFIX}-restore-${{inputs.kind }}.log ./corso restore '${{ inputs.service }}' \ --no-stats \ --hide-progress \ @@ -91,11 +102,17 @@ runs: SANITY_TEST_RESTORE_CONTAINER: ${{ steps.restore.outputs.result }} SANITY_TEST_SOURCE_CONTAINER: ${{ inputs.restore-container }} SANITY_BACKUP_ID: ${{ inputs.backup-id }} + # lists are not restored to a different folder. they get created adjacent to their originals + # hence SANITY_TEST_RESTORE_CONTAINER_PREFIX is necessary to differentiate restored from original + SANITY_TEST_RESTORE_CONTAINER_PREFIX: ${{ steps.restore.outputs.result }} + SANITY_TEST_CATEGORY: ${{ inputs.category }} run: | echo "---------------------------" echo Sanity Test Restore ${{ inputs.service }} ${{ inputs.kind }} echo "---------------------------" - CORSO_LOG_FILE=${{ inputs.log-dir }}/gotest-${{ inputs.service }}-validate-${{inputs.kind }}.log + CATEGORY_SUFFIX="" + [[ -n "${{ inputs.category }}" ]] && CATEGORY_SUFFIX="-${{ inputs.category }}" + CORSO_LOG_FILE=${{ inputs.log-dir }}/gotest-${{ inputs.service }}${CATEGORY_SUFFIX}-validate-${{inputs.kind }}.log ./sanity-test restore ${{ inputs.service }} - name: Export ${{ inputs.service }} ${{ inputs.kind }} @@ -108,9 +125,11 @@ runs: echo Export ${{ inputs.service }} ${{ inputs.kind }} echo "---------------------------" set -euo pipefail - CORSO_LOG_FILE=${{ inputs.log-dir }}/gotest-${{ inputs.service }}-restore-${{inputs.kind }}.log + CATEGORY_SUFFIX="" + [[ -n "${{ inputs.category }}" ]] && CATEGORY_SUFFIX="-${{ inputs.category }}" + CORSO_LOG_FILE=${{ inputs.log-dir }}/gotest-${{ inputs.service }}${CATEGORY_SUFFIX}-restore-${{inputs.kind }}.log ./corso export '${{ inputs.service }}' \ - /tmp/export-${{ inputs.service }}-${{inputs.kind }} \ + /tmp/export-${{ inputs.service }}${CATEGORY_SUFFIX}-${{inputs.kind }} \ --no-stats \ --hide-progress \ ${{ inputs.export-args }} \ @@ -123,14 +142,19 @@ runs: shell: bash working-directory: src env: - SANITY_TEST_RESTORE_CONTAINER: /tmp/export-${{ inputs.service }}-${{inputs.kind }} + SANITY_TEST_RESTORE_CONTAINER: /tmp/export-${{ inputs.service }}${{ inputs.category && '-' }}${{ inputs.category }}-${{ inputs.kind }} SANITY_TEST_SOURCE_CONTAINER: ${{ inputs.restore-container }} SANITY_BACKUP_ID: ${{ inputs.backup-id }} + # applies only for sharepoint lists + SANITY_TEST_RESTORE_CONTAINER_PREFIX: ${{ steps.restore.outputs.result }} + SANITY_TEST_CATEGORY: ${{ inputs.category }} run: | echo "---------------------------" echo Sanity-Test Export ${{ inputs.service }} ${{ inputs.kind }} echo "---------------------------" - CORSO_LOG_FILE=${{ inputs.log-dir }}/gotest-${{ inputs.service }}-validate-${{inputs.kind }}.log + CATEGORY_SUFFIX="" + [[ -n "${{ inputs.category }}" ]] && CATEGORY_SUFFIX="-${{ inputs.category }}" + CORSO_LOG_FILE=${{ inputs.log-dir }}/gotest-${{ inputs.service }}${CATEGORY_SUFFIX}-validate-${{inputs.kind }}.log ./sanity-test export ${{ inputs.service }} - name: Export archive ${{ inputs.service }} ${{ inputs.kind }} @@ -143,17 +167,19 @@ runs: echo Export Archive ${{ inputs.service }} ${{ inputs.kind }} echo "---------------------------" set -euo pipefail - CORSO_LOG_FILE=${{ inputs.log-dir }}/gotest-${{ inputs.service }}-restore-${{inputs.kind }}.log + CATEGORY_SUFFIX="" + [[ -n "${{ inputs.category }}" ]] && CATEGORY_SUFFIX="-${{ inputs.category }}" + CORSO_LOG_FILE=${{ inputs.log-dir }}/gotest-${{ inputs.service }}${CATEGORY_SUFFIX}-restore-${{inputs.kind }}.log ./corso export '${{ inputs.service }}' \ - /tmp/export-${{ inputs.service }}-${{inputs.kind }}-archive \ + /tmp/export-${{ inputs.service }}${CATEGORY_SUFFIX}-${{inputs.kind }}-archive \ --no-stats \ --hide-progress \ --archive \ ${{ inputs.export-args }} \ --backup '${{ steps.backup.outputs.result }}' - unzip /tmp/export-${{ inputs.service }}-${{inputs.kind }}-archive/*.zip \ - -d /tmp/export-${{ inputs.service }}-${{inputs.kind }}-unzipped + unzip /tmp/export-${{ inputs.service }}${CATEGORY_SUFFIX}-${{inputs.kind }}-archive/*.zip \ + -d /tmp/export-${{ inputs.service }}${CATEGORY_SUFFIX}-${{inputs.kind }}-unzipped cat /tmp/corsologs - name: Check archive export ${{ inputs.service }} ${{ inputs.kind }} @@ -161,14 +187,19 @@ runs: shell: bash working-directory: src env: - SANITY_TEST_RESTORE_CONTAINER: /tmp/export-${{ inputs.service }}-${{inputs.kind }}-unzipped + SANITY_TEST_RESTORE_CONTAINER: /tmp/export-${{ inputs.service }}${{ inputs.category && '-' }}${{ inputs.category }}-${{inputs.kind }}-unzipped SANITY_TEST_SOURCE_CONTAINER: ${{ inputs.restore-container }} SANITY_BACKUP_ID: ${{ inputs.backup-id }} + # applies only for sharepoint lists + SANITY_TEST_RESTORE_CONTAINER_PREFIX: ${{ steps.restore.outputs.result }} + SANITY_TEST_CATEGORY: ${{ inputs.category }} run: | echo "---------------------------" echo Sanity-Test Export Archive ${{ inputs.service }} ${{ inputs.kind }} echo "---------------------------" - CORSO_LOG_FILE=${{ inputs.log-dir }}/gotest-${{ inputs.service }}-validate-${{inputs.kind }}.log + CATEGORY_SUFFIX="" + [[ -n "${{ inputs.category }}" ]] && CATEGORY_SUFFIX="-${{ inputs.category }}" + CORSO_LOG_FILE=${{ inputs.log-dir }}/gotest-${{ inputs.service }}${CATEGORY_SUFFIX}-validate-${{inputs.kind }}.log ./sanity-test export ${{ inputs.service }} - name: List ${{ inputs.service }} ${{ inputs.kind }} @@ -179,7 +210,9 @@ runs: echo Backup list ${{ inputs.service }} ${{ inputs.kind }} echo "---------------------------" set -euo pipefail - CORSO_LOG_FILE=${{ inputs.log-dir }}/gotest-backup-${{ inputs.service }}-list-${{inputs.kind }}.log + CATEGORY_SUFFIX="" + [[ -n "${{ inputs.category }}" ]] && CATEGORY_SUFFIX="-${{ inputs.category }}" + CORSO_LOG_FILE=${{ inputs.log-dir }}/gotest-backup-${{ inputs.service }}${CATEGORY_SUFFIX}-list-${{inputs.kind }}.log ./corso backup list ${{ inputs.service }} \ --no-stats \ --hide-progress \ @@ -200,7 +233,10 @@ runs: echo Backup List w/ Backup ${{ inputs.service }} ${{ inputs.kind }} echo "---------------------------" set -euo pipefail - CORSO_LOG_FILE=${{ inputs.log-dir }}/gotest-backup-list-${{ inputs.service }}-single-${{inputs.kind }}.log + # Include category in the log file name if present + CATEGORY_SUFFIX="" + [[ -n "${{ inputs.category }}" ]] && CATEGORY_SUFFIX="-${{ inputs.category }}" + CORSO_LOG_FILE=${{ inputs.log-dir }}/gotest-backup-list-${{ inputs.service }}${CATEGORY_SUFFIX}-single-${{inputs.kind }}.log ./corso backup list ${{ inputs.service }} \ --no-stats \ --hide-progress \ @@ -228,4 +264,4 @@ runs: name: "${{ inputs.service }}-${{ inputs.kind }}-logs" path: ${{ inputs.log-dir }}/* if-no-files-found: error - retention-days: 14 \ No newline at end of file + retention-days: 14 diff --git a/.github/workflows/sanity-test.yaml b/.github/workflows/sanity-test.yaml index 70b0b7a1b..c2c960907 100644 --- a/.github/workflows/sanity-test.yaml +++ b/.github/workflows/sanity-test.yaml @@ -302,7 +302,7 @@ jobs: ########################################################################################################################################## -# Sharepoint +# Sharepoint Library # generate new entries for test - name: SharePoint - Create new data @@ -329,11 +329,12 @@ jobs: with: service: sharepoint kind: first-backup - backup-args: '--site "${{ vars.CORSO_M365_TEST_SITE_URL }}"' + backup-args: '--site "${{ vars.CORSO_M365_TEST_SITE_URL }}" --data libraries' restore-args: '--folder ${{ env.RESTORE_DEST_PFX }}${{ steps.new-data-creation-sharepoint.outputs.result }}' restore-container: '${{ env.RESTORE_DEST_PFX }}${{ steps.new-data-creation-sharepoint.outputs.result }}' log-dir: ${{ env.CORSO_LOG_DIR }} with-export: true + category: libraries # generate some more enteries for incremental check - name: SharePoint - Create new data (for incremental) @@ -355,11 +356,103 @@ jobs: with: service: sharepoint kind: incremental - backup-args: '--site "${{ vars.CORSO_M365_TEST_SITE_URL }}"' + backup-args: '--site "${{ vars.CORSO_M365_TEST_SITE_URL }}" --data libraries' restore-args: '--folder ${{ env.RESTORE_DEST_PFX }}${{ steps.new-data-creation-sharepoint.outputs.result }}' restore-container: '${{ env.RESTORE_DEST_PFX }}${{ steps.new-data-creation-sharepoint.outputs.result }}' log-dir: ${{ env.CORSO_LOG_DIR }} with-export: true + category: libraries + +########################################################################################################################################## + +# Sharepoint Lists + + # generate new entries for test + # The `awk | tr | sed` command chain is used to get a comma separated list of SharePoint list names. + - name: SharePoint Lists - Create new data + id: new-data-creation-sharepoint-lists + timeout-minutes: 30 + working-directory: ./src/cmd/factory + run: | + suffix=$(date +"%Y-%m-%d_%H-%M-%S") + + go run . sharepoint lists \ + --site ${{ vars.CORSO_M365_TEST_SITE_URL }} \ + --user ${{ env.TEST_USER }} \ + --secondaryuser ${{ env.CORSO_SECONDARY_M365_TEST_USER_ID }} \ + --tenant ${{ secrets.TENANT_ID }} \ + --destination ${{ env.RESTORE_DEST_PFX }}$suffix \ + --count 4 | + awk 'NR > 1 {print $2}' | tr '\n' ',' | sed -e 's/,$//' -e 's/^/result=/' | + tee $GITHUB_OUTPUT + # Extracts the common prefix for the Sharepoint list names. + - name: SharePoint Lists - Store restore container + id: sharepoint-lists-store-restore-container + run: | + echo ${{ steps.new-data-creation-sharepoint-lists.outputs.result }} | + cut -d',' -f1 | + cut -d'_' -f1,2,3,4,5 | + sed -e 's/^/result=/' | + tee $GITHUB_OUTPUT + + - name: SharePoint Lists - Backup + id: sharepoint-lists-backup + timeout-minutes: 30 + uses: ./.github/actions/backup-restore-test + with: + service: sharepoint + kind: first-backup-lists + backup-args: '--site "${{ vars.CORSO_M365_TEST_SITE_URL }}" --data lists' + restore-args: "--list ${{ steps.new-data-creation-sharepoint-lists.outputs.result }} --destination Corso_Test_Sanity_Restore_$(date +'%Y%m%d_%H%M%S') --allow-lists-restore" + export-args: "--list ${{ steps.new-data-creation-sharepoint-lists.outputs.result }}" + restore-container: "${{ steps.sharepoint-lists-store-restore-container.outputs.result }}" + log-dir: ${{ env.CORSO_LOG_DIR }} + with-export: true + category: lists + on-collision: copy + + # generate some more enteries for incremental check + - name: SharePoint Lists - Create new data (for incremental) + id: inc-data-creation-sharepoint-lists + timeout-minutes: 30 + working-directory: ./src/cmd/factory + run: | + suffix=$(date +"%Y-%m-%d_%H-%M-%S") + + go run . sharepoint lists \ + --site ${{ vars.CORSO_M365_TEST_SITE_URL }} \ + --user ${{ env.TEST_USER }} \ + --secondaryuser ${{ env.CORSO_SECONDARY_M365_TEST_USER_ID }} \ + --tenant ${{ secrets.TENANT_ID }} \ + --destination ${{ env.RESTORE_DEST_PFX }}$suffix \ + --count 4 | + awk 'NR > 1 {print $2}' | tr '\n' ',' | sed -e 's/,$//' -e 's/^/result=/' | + tee $GITHUB_OUTPUT + + - name: SharePoint Lists - Store restore container (for incremental) + id: sharepoint-lists-store-restore-container-inc + run: | + echo ${{ steps.inc-data-creation-sharepoint-lists.outputs.result }} | + cut -d',' -f1 | + cut -d'_' -f1,2,3,4,5 | + sed -e 's/^/result=/' | + tee $GITHUB_OUTPUT + + - name: SharePoint Lists - Incremental backup + id: sharepoint-lists-incremental + timeout-minutes: 30 + uses: ./.github/actions/backup-restore-test + with: + service: sharepoint + kind: incremental-lists + backup-args: '--site "${{ vars.CORSO_M365_TEST_SITE_URL }}" --data lists' + restore-args: "--list ${{ steps.inc-data-creation-sharepoint-lists.outputs.result }},${{ steps.new-data-creation-sharepoint-lists.outputs.result }} --destination Corso_Test_Sanity_Restore_$(date +'%Y%m%d_%H%M%S') --allow-lists-restore" + export-args: "--list ${{ steps.inc-data-creation-sharepoint-lists.outputs.result }},${{ steps.new-data-creation-sharepoint-lists.outputs.result }}" + restore-container: "${{ steps.sharepoint-lists-store-restore-container-inc.outputs.result }},${{ steps.sharepoint-lists-store-restore-container.outputs.result }}" + log-dir: ${{ env.CORSO_LOG_DIR }} + with-export: true + category: lists + on-collision: copy ##########################################################################################################################################