diff --git a/.github/actions/publish-website/action.yml b/.github/actions/publish-website/action.yml new file mode 100644 index 000000000..33be7555a --- /dev/null +++ b/.github/actions/publish-website/action.yml @@ -0,0 +1,46 @@ +name: Publish Website + +inputs: + aws-iam-role: + description: IAM role for connecting to AWS + bucket: + description: Bucket to push the website files + cfid: + description: Cloudfront ID for invalidation + +runs: + using: composite + steps: + - uses: actions/checkout@v3 + + - uses: actions/download-artifact@master + name: Download website from build step + with: + name: website + path: website/build + + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v1 + with: + role-to-assume: ${{ inputs.aws-iam-role }} + role-session-name: integration-testing + aws-region: us-east-1 + + - name: Add rotbots.txt + if: github.ref == 'refs/heads/asset-validation-explore' + working-directory: website + shell: bash + run: | + printf 'User-agent: *\nDisallow: /' > build/robots.txt + + - name: Push website + working-directory: website + shell: bash + run: | + aws s3 sync build "s3://${{ inputs.bucket }}" --delete + + - name: Invalidate cloudfront + working-directory: website + shell: bash + run: | + aws cloudfront create-invalidation --distribution-id ${{ inputs.cfid }} --paths "/*" \ No newline at end of file diff --git a/.github/actions/website-linting/action.yml b/.github/actions/website-linting/action.yml new file mode 100644 index 000000000..3e04acacf --- /dev/null +++ b/.github/actions/website-linting/action.yml @@ -0,0 +1,57 @@ +name: Lint Website + +inputs: + version: + description: Corso version to use for publishing + +runs: + using: composite + steps: + - uses: actions/checkout@v3 + + - name: Setup Golang with cache + uses: magnetikonline/action-golang-cache@v3 + with: + go-version-file: src/go.mod + + - name: Generate CLI Docs + working-directory: ./src + shell: bash + run: | + go run ./cmd/mdgen/mdgen.go generate + + # migrate generated md files into /website/docs/cli + - name: Move CLI .md to Docs + shell: bash + run: | + mkdir -p ./website/docs/cli + mv ./src/cmd/mdgen/cli_markdown/* ./website/docs/cli/ + rm -R ./src/cmd/mdgen/cli_markdown/ + + - name: Install dependencies for website lint + shell: bash + run: | + wget --quiet https://github.com/errata-ai/vale/releases/download/v2.20.2/vale_2.20.2_Linux_64-bit.tar.gz # NOTE: update in Dockerfile when updating + mkdir bin && tar -xvzf vale_2.20.2_Linux_64-bit.tar.gz -C bin + echo "$PWD/bin" >> $GITHUB_PATH + npm i -g markdownlint-cli@0.32.2 # NOTE: update in Dockerfile when updating + + - name: Run website lint + shell: bash + run: | + cd website && make -o genclidocs localcheck + + - name: Build website + shell: bash + env: + CORSO_VERSION: ${{ inputs.version }} + run: | + cd website && + npm ci && + npm run build + + - uses: actions/upload-artifact@master + name: Upload website as artifacts + with: + name: website + path: website/build \ No newline at end of file diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c1a022e5d..e55b913e8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,4 +1,6 @@ name: Build/Release Corso +env: + IMAGE_NAME: ghcr.io/alcionai/corso on: workflow_dispatch: @@ -103,6 +105,7 @@ jobs: echo "::set-output name=website-cfid::ESFTEIYTIP7Y3" fi + # ---------------------------------------------------------------------------------------------------- # --- Website Linting ----------------------------------------------------------------------------------- # ---------------------------------------------------------------------------------------------------- @@ -116,47 +119,10 @@ jobs: steps: - uses: actions/checkout@v3 - - name: Setup Golang with cache - uses: magnetikonline/action-golang-cache@v3 + - name: Lint Website + uses: ./.github/actions/website-linting with: - go-version-file: src/go.mod - - - name: Generate CLI Docs - working-directory: ./src - run: | - go run ./cmd/mdgen/mdgen.go generate - - # migrate generated md files into /website/docs/cli - - name: Move CLI .md to Docs - run: | - mkdir -p ./website/docs/cli - mv ./src/cmd/mdgen/cli_markdown/* ./website/docs/cli/ - rm -R ./src/cmd/mdgen/cli_markdown/ - - - name: Install dependencies for website lint - run: | - wget https://github.com/errata-ai/vale/releases/download/v2.20.2/vale_2.20.2_Linux_64-bit.tar.gz # NOTE: update in Dockerfile when updating - mkdir bin && tar -xvzf vale_2.20.2_Linux_64-bit.tar.gz -C bin - echo "$PWD/bin" >> $GITHUB_PATH - npm i -g markdownlint-cli@0.32.2 # NOTE: update in Dockerfile when updating - - - name: Run website lint - run: | - cd website && make -o genclidocs localcheck - - - name: Build website - env: - CORSO_VERSION: ${{ needs.SetEnv.outputs.version }} - run: | - cd website && - npm ci && - npm run build - - - uses: actions/upload-artifact@master - name: Upload website as artifacts - with: - name: website - path: website/build + version: ${{ needs.SetEnv.outputs.version }} # ---------------------------------------------------------------------------------------------------- # --- Integration and Unit Testing ------------------------------------------------------------------- @@ -408,44 +374,6 @@ jobs: name: corso path: src/dist/* - Publish-Website: - needs: [Test-Suite-Trusted, Linting, Website-Linting, SetEnv] - environment: ${{ needs.SetEnv.outputs.environment }} - runs-on: ubuntu-latest - if: startsWith(github.ref, 'refs/tags/') || github.ref == 'refs/heads/main' - defaults: - run: - working-directory: website - - steps: - - uses: actions/checkout@v3 - - - uses: actions/download-artifact@master - name: Download website from build step - with: - name: website - path: website/build - - - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@v1 - with: - role-to-assume: ${{ secrets.AWS_IAM_ROLE }} - role-session-name: integration-testing - aws-region: us-east-1 - - - name: Add rotbots.txt - if: github.ref == 'refs/heads/main' - run: | - printf 'User-agent: *\nDisallow: /' > build/robots.txt - - - name: Push website - run: | - aws s3 sync build "s3://${{ needs.SetEnv.outputs.website-bucket }}" --delete - - - name: Invalidate cloudfront - run: | - aws cloudfront create-invalidation --distribution-id ${{ needs.SetEnv.outputs.website-cfid }} --paths "/*" - Publish-Image: needs: [Test-Suite-Trusted, Linting, Website-Linting, SetEnv] environment: ${{ needs.SetEnv.outputs.environment }} @@ -455,7 +383,6 @@ jobs: run: working-directory: build env: - imageName: ghcr.io/alcionai/corso PLATFORMS: linux/amd64,linux/arm64 steps: - uses: actions/checkout@v3 @@ -478,7 +405,7 @@ jobs: id: meta uses: docker/metadata-action@v4 with: - images: ${{ env.imageName }} + images: ${{ env.IMAGE_NAME }} tags: | type=ref,event=tag type=sha,format=short,prefix= @@ -499,3 +426,136 @@ jobs: # use the github cache cache-from: type=gha cache-to: type=gha,mode=max + + Validate-Linux-Artifacts: + needs: [Publish-Binary, Publish-Image, SetEnv] + environment: Testing + runs-on: ubuntu-latest + if: startsWith(github.ref, 'refs/tags/') + env: + CORSO_VERSION: ${{ needs.SetEnv.outputs.version }} + steps: + - name: Validate x86_64 binary artifacts + run: | + set -ex + curl -L https://github.com/alcionai/corso/releases/download/${{ env.CORSO_VERSION }}/corso_${{ env.CORSO_VERSION }}_Linux_x86_64.tar.gz > corso.tar.gz + tar -xf corso.tar.gz + ./corso --help + ./corso --version 2>&1 | grep -E "^version: ${{ env.CORSO_VERSION }}$" + - name: Validate arm64 binary artifacts + uses: uraimo/run-on-arch-action@v2 + with: + arch: armv7 + distro: ubuntu18.04 + install: | + apt-get -y update && apt-get -y install curl + run: | + set -ex + sudo apt-get update && sudo apt-get install curl + curl -L https://github.com/alcionai/corso/releases/download/${{ env.CORSO_VERSION }}/corso_${{ env.CORSO_VERSION }}_Linux_arm64.tar.gz > corso.tar.gz + tar -xf corso.tar.gz + ./corso --help + ./corso --version 2>&1 | grep -E "^version: ${{ env.CORSO_VERSION }}$" + + Validate-Docker-Artifacts: + needs: [Publish-Binary, Publish-Image, SetEnv] + environment: Testing + runs-on: ubuntu-latest + if: startsWith(github.ref, 'refs/tags/') + env: + CORSO_VERSION: ${{ needs.SetEnv.outputs.version }} + steps: + - name: Set up QEMU + uses: docker/setup-qemu-action@v2 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + + - name: Validate amd64 container images + run: | + docker run --platform linux/amd64 ${{ env.IMAGE_NAME }}:${{ env.CORSO_VERSION }} --help + docker run --platform linux/amd64 ${{ env.IMAGE_NAME }}:${{ env.CORSO_VERSION }} --version | grep -E "^version: ${{ env.CORSO_VERSION }}$" + - name: Validate arm64 container images + run: | + docker run --platform linux/arm64 ${{ env.IMAGE_NAME }}:${{ env.CORSO_VERSION }} --help + docker run --platform linux/amd64 ${{ env.IMAGE_NAME }}:${{ env.CORSO_VERSION }} --version | grep -E "^version: ${{ env.CORSO_VERSION }}$" + + Validate-MacOS-Artifacts: + needs: [Publish-Binary, Publish-Image, SetEnv] + environment: Testing + runs-on: macos-latest + if: startsWith(github.ref, 'refs/tags/') + env: + CORSO_VERSION: ${{ needs.SetEnv.outputs.version }} + steps: + - name: Validate x86_64 binary artifacts + run: | + set -ex + curl -L https://github.com/alcionai/corso/releases/download/${{ env.CORSO_VERSION }}/corso_${{ env.CORSO_VERSION }}_Darwin_x86_64.tar.gz > corso.tar.gz + tar -xf corso.tar.gz + ./corso --help + ./corso --version 2>&1 | grep -E "^version: ${{ env.CORSO_VERSION }}$" + - name: Validate arm64 binary artifacts + run: | + set -ex + curl -L https://github.com/alcionai/corso/releases/download/${{ env.CORSO_VERSION }}/corso_${{ env.CORSO_VERSION }}_Darwin_arm64.tar.gz > corso.tar.gz + tar -xf corso.tar.gz + # TODO: test/validate arm64 executable once we have arm64 CI + + Validate-Windows-Artifacts: + needs: [Publish-Binary, Publish-Image, SetEnv] + environment: Testing + runs-on: windows-latest + if: startsWith(github.ref, 'refs/tags/') + env: + CORSO_VERSION: ${{ needs.SetEnv.outputs.version }} + steps: + - name: Validate x86_64 binary artifacts + run: | + curl -L https://github.com/alcionai/corso/releases/download/${{ env.CORSO_VERSION }}/corso_${{ env.CORSO_VERSION }}_Windows_x86_64.zip -o corso.zip + 7z x corso.zip + ./corso.exe --help + ./corso.exe --version 2>&1 | grep -E "^version: ${{ env.CORSO_VERSION }}$" + + Publish-Website-Test: + needs: [Test-Suite-Trusted, Linting, Website-Linting, SetEnv] + environment: ${{ needs.SetEnv.outputs.environment }} + runs-on: ubuntu-latest + if: github.ref == 'refs/heads/main' + + steps: + - uses: actions/checkout@v3 # need to checkout to make the action available + + - name: Publish website + uses: ./.github/actions/publish-website + with: + aws-iam-role: ${{ secrets.AWS_IAM_ROLE }} + bucket: ${{ needs.SetEnv.outputs.website-bucket }} + cfid: ${{ needs.SetEnv.outputs.website-cfid }} + + Publish-Website-Prod: + needs: [SetEnv, Validate-Linux-Artifacts, Validate-MacOS-Artifacts, Validate-Docker-Artifacts, Validate-Windows-Artifacts] + environment: ${{ needs.SetEnv.outputs.environment }} + runs-on: ubuntu-latest + if: startsWith(github.ref, 'refs/tags/') + + steps: + - uses: actions/checkout@v3 # need to checkout to make the action available + + - name: Publish website + uses: ./.github/actions/publish-website + with: + aws-iam-role: ${{ secrets.AWS_IAM_ROLE }} + bucket: ${{ needs.SetEnv.outputs.website-bucket }} + cfid: ${{ needs.SetEnv.outputs.website-cfid }} + + Validate-Website-Artifacts: + needs: [Publish-Website-Prod, SetEnv] + environment: Testing + runs-on: ubuntu-latest + if: startsWith(github.ref, 'refs/tags/') + env: + CORSO_VERSION: ${{ needs.SetEnv.outputs.version }} + steps: + - name: Validate docs + run: | + curl https://corsobackup.io/docs/quickstart/ | grep https://github.com/alcionai/corso/releases/download/${{ env.CORSO_VERSION }}/corso_${{ env.CORSO_VERSION }}_Linux_x86_64.tar.gz \ No newline at end of file diff --git a/.github/workflows/website-publish.yml b/.github/workflows/website-publish.yml index 6a8b7f285..b53ed320d 100644 --- a/.github/workflows/website-publish.yml +++ b/.github/workflows/website-publish.yml @@ -43,47 +43,10 @@ jobs: steps: - uses: actions/checkout@v3 - - name: Setup Golang with cache - uses: magnetikonline/action-golang-cache@v3 + - name: Lint Website + uses: ./.github/actions/website-linting with: - go-version-file: src/go.mod - - - name: Generate CLI Docs - working-directory: ./src - run: | - go run ./cmd/mdgen/mdgen.go generate - - # migrate generated md files into /website/docs/cli - - name: Move CLI .md to Docs - run: | - mkdir -p ./website/docs/cli - mv ./src/cmd/mdgen/cli_markdown/* ./website/docs/cli/ - rm -R ./src/cmd/mdgen/cli_markdown/ - - - name: Install dependencies for website lint - run: | - wget https://github.com/errata-ai/vale/releases/download/v2.20.2/vale_2.20.2_Linux_64-bit.tar.gz # NOTE: update in Dockerfile when updating - mkdir bin && tar -xvzf vale_2.20.2_Linux_64-bit.tar.gz -C bin - echo "$PWD/bin" >> $GITHUB_PATH - npm i -g markdownlint-cli@0.32.2 # NOTE: update in Dockerfile when updating - - - name: Run website lint - run: | - cd website && make -o genclidocs localcheck - - - name: Build website - env: - CORSO_VERSION: ${{ needs.SetEnv.outputs.version }} - run: | - cd website && - npm ci && - npm run build - - - uses: actions/upload-artifact@master - name: Upload website as artifacts - with: - name: website - path: website/build + version: ${{ needs.SetEnv.outputs.version }} Publish-Website: needs: [Website-Linting] @@ -94,25 +57,11 @@ jobs: working-directory: website steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v3 # need to checkout to make the action available - - uses: actions/download-artifact@master - name: Download website from build step + - name: Publish website + uses: ./.github/actions/publish-website with: - name: website - path: website/build - - - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@v1 - with: - role-to-assume: ${{ secrets.AWS_IAM_ROLE }} - role-session-name: integration-testing - aws-region: us-east-1 - - - name: Push website - run: | - aws s3 sync build "s3://corsobackup.io" --delete - - - name: Invalidate cloudfront - run: | - aws cloudfront create-invalidation --distribution-id E1W9NGI9YTVZ1A --paths "/*" \ No newline at end of file + aws-iam-role: ${{ secrets.AWS_IAM_ROLE }} + bucket: "corsobackup.io" + cfid: "E1W9NGI9YTVZ1A" \ No newline at end of file