Issue 570 deploy sep b (#719)

Recovery action after losing this commit upon merging `issue-570-deploy`.

.github/workflows/container.yml

@@ -1,114 +0,0 @@
-name: Publish Docker Container Images
-on:
-  push:
-    branches: [main]
-
-env:
-  REGISTRY: ghcr.io
-  REPO_NAME: ${{ github.repository }}
-
-permissions:
-  contents: read
-  packages: write
-
-jobs:
-
-  # ------------------------------------------------------------------------------------------
-  # To be decided: Script-Deploy or Dockerfile-Deploy:
-  # Script:
-  #   + Separates the golang build from the corso build.
-  #   - Haven't figured out multiplatform builds yet.
-  #   - Doesn't cache, always takes 10-15 minutes per build in the matrix.
-  # Dockerfile:
-  #   + Once cached, takes <1m to deploy.
-  #   + Multiplatform.
-  #   + Extended features (such as tagging) can be handled by more github actions.
-  #   - When not cached, can take >2 hours to build (at least initially).
-  #   - Currently includes the complete golang:1.18 image.
-  # ------------------------------------------------------------------------------------------
-
-  Script-Deploy:
-    runs-on: ubuntu-latest
-    defaults:
-      run:
-        working-directory: build
-    strategy:
-      matrix:
-        BUILD_ARCH: [amd64, arm64]
-        BUILD_OS: [linux]
-    env:
-      IMAGE_PREFIX: ghcr.io
-      VERSION_SUFFIX: rolling
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v3
-
-      - name: Run build script
-        run: >
-          ./build-container.sh
-          --arch ${{ matrix.BUILD_ARCH }}
-          --prefix ${{ env.IMAGE_PREFIX }}
-          --suffix ${{ env.VERSION_SUFFIX }}
-
-      # login step boilerplate from:
-      # https://docs.github.com/en/packages/managing-github-packages-using-github-actions-workflows/publishing-and-installing-a-package-with-github-actions#upgrading-a-workflow-that-accesses-ghcrio
-      - name: Log in to registry
-        run: echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u $ --password-stdin
-
-      - name: Push image
-        env:
-          IMAGE_ID: ${{ env.IMAGE_PREFIX }}/alcionai/corso
-          VERSION: ${{ matrix.BUILD_OS }}-${{ matrix.BUILD_ARCH }}-${{ env.VERSION_SUFFIX }}
-        run: |
-          docker images -a
-          docker push ${{ env.IMAGE_ID }}:${{ env.VERSION }}
-
-  Dockerfile-Deploy:
-    runs-on: ubuntu-latest
-    env:
-      TARGETOS: linux
-      TARGETARCH: arm64
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v3
-      
-      # apparently everyone uses this step
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@v2
-
-      # setup Docker buld action
-      - name: Set up Docker Buildx
-        id: buildx
-        uses: docker/setup-buildx-action@v2
-
-      # In case we want to switch to dockerhub      
-      # - name: Login to DockerHub
-      #   uses: docker/login-action@v2
-      #   with:
-      #     username: ${{ secrets.DOCKERHUB_USERNAME }}
-      #     password: ${{ secrets.DOCKERHUB_TOKEN }}
-
-      # retrieve credentials for ghcr.io
-      - name: Login to Github Packages
-        uses: docker/login-action@v2
-        with:
-          registry: ghcr.io
-          username: ${{ github.actor }}
-          password: ${{ secrets.GITHUB_TOKEN }}
-
-      # build the image
-      - name: Build image and push to Docker Hub and GitHub Container Registry
-        uses: docker/build-push-action@v3
-        with:
-          context: .
-          file: ./docker/Dockerfile
-          platforms: linux/amd64,linux/arm64
-          push: true
-          tags: ghcr.io/alcionai/corso:rolling
-          # use the github cache
-          cache-from: type=gha
-          cache-to: type=gha,mode=max
-
-      # check the image digest
-      - name: Image digest
-        run: echo ${{ steps.docker_build.outputs.digest }}

.github/workflows/image.yml

@@ -0,0 +1,66 @@
+name: Publish Docker Container Images
+on:
+  push:
+    branches: [main]
+
+permissions:
+  contents: read
+  packages: write
+
+jobs:
+  Per-SHA-Image:
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: build
+    env:
+      PLATFORMS: linux/amd64,linux/arm64
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+
+      - name: Build Corso Binaries
+        run: >
+          ./build.sh
+          --platforms ${{ env.PLATFORMS }}
+
+      # - name: Build Corso Binaries Locally
+      #   run: >
+      #     ./multiplatform-binary.sh
+      #     --platforms ${{ env.PLATFORMS }}
+
+      # apparently everyone uses this step
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v2
+
+      # setup Docker buld action
+      - name: Set up Docker Buildx
+        id: buildx
+        uses: docker/setup-buildx-action@v2
+
+      # retrieve credentials for ghcr.io
+      - name: Login to Github Packages
+        uses: docker/login-action@v2
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      # deploy the image
+      - name: Build image and push to GitHub Container Registry
+        env:
+          sha: git_hash=$(git rev-parse --short "$GITHUB_SHA")
+        uses: docker/build-push-action@v3
+        with:
+          context: .
+          file: ./build/Dockerfile
+          platforms: ${{ env.PLATFORMS }}
+          push: true
+          tags: ghcr.io/alcionai/corso:${{ env.sha }}
+          # use the github cache
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+
+      # check the image digest
+      - name: Image digest
+        run: echo ${{ steps.docker_build.outputs.digest }}

build/Dockerfile

@@ -8,7 +8,9 @@ FROM gcr.io/distroless/base-debian10
 
 WORKDIR /
 
-COPY ./bin/corso ./
+ARG TARGETOS
+ARG TARGETARCH
+COPY ./bin/${TARGETOS}-${TARGETARCH}/corso ./
 
 USER nonroot:nonroot
 

build/build-container.sh

@@ -11,6 +11,7 @@ usage() {
   echo "Flags"
   echo "  -h|--help         Help"
   echo "  -a|--arch         Set the architecture to the specified value (default: amd64)"
+  echo "  -l|--local        Build the corso binary on your local system, rather than a go image"
   echo "  -p|--prefix       Prefixes the image name."
   echo "  -s|--suffix       Suffixes the version."
   echo " "
@@ -30,6 +31,7 @@ OS=linux
 ARCH=amd64
 IMAGE_NAME_PREFIX=
 IMAGE_TAG_SUFFIX=
+LOCAL=
 
 while [ "$#" -gt 0 ]
 do
@@ -42,6 +44,9 @@ do
     ARCH=$2
     shift
     ;;
+  -l|--local)
+    LOCAL=1
+    ;;
   -p|--prefix)
     IMAGE_NAME_PREFIX=$2
     shift
@@ -62,6 +67,8 @@ do
   shift
 done
 
+TARGETPLATFORM=${OS}/${ARCH}
+
 IMAGE_TAG=${OS}-${ARCH}
 if [ ! -z "${IMAGE_TAG_SUFFIX}" ]; then
   IMAGE_TAG=${IMAGE_TAG}-${IMAGE_TAG_SUFFIX}
@@ -72,7 +79,11 @@ if [ ! -z "${IMAGE_NAME_PREFIX}" ]; then
   IMAGE_NAME=${IMAGE_NAME_PREFIX}/${IMAGE_NAME}
 fi
 
-${SCRIPT_ROOT}/build.sh --arch ${ARCH}
+if [ -z "$LOCAL" ]; then
+  ${SCRIPT_ROOT}/build.sh --platforms "${TARGETPLATFORM}"
+else
+  ${SCRIPT_ROOT}/multiplatform-binary.sh --platforms "${TARGETPLATFORM}"
+fi
 
 echo "-----"
 echo "building corso container ${IMAGE_NAME}"
@@ -80,8 +91,8 @@ echo "-----"
 
 set -x
 docker buildx build --tag ${IMAGE_NAME}     \
-  --platform ${OS}/${ARCH}                  \
+  --platform ${TARGETPLATFORM}              \
-  --file ${PROJECT_ROOT}/build/Dockerfile  \
+  --file ${PROJECT_ROOT}/build/Dockerfile   \
   ${PROJECT_ROOT}
 set +x
 

build/build.sh

@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/bin/bash
 
 set -e
 
@@ -16,6 +16,7 @@ CORSO_MOD_CACHE=${CORSO_BUILD_PKG_MOD}/cache
 
 CORSO_BUILD_ARGS=''
 
+platforms=
 GOVER=1.18
 GOOS=linux
 GOARCH=amd64
@@ -23,8 +24,8 @@ GOARCH=amd64
 while [ "$#" -gt 0 ]
 do
   case "$1" in
-  --arch)
+  --platforms)
-    GOARCH=$2
+    platforms=$2
     shift
     ;;
   esac
@@ -36,27 +37,39 @@ mkdir -p ${CORSO_BUILD_TMP_CACHE}
 # temporary directory for caching go modules (needed for fast cross-platform build)
 mkdir -p ${CORSO_BUILD_TMP_MOD}
 
-echo "-----"
+if [ -z "$platforms" ]; then
-echo "building corso binary for ${GOOS}-${GOARCH}"
+  platforms="${GOOS}/${GOARCH}"
-echo "-----"
+fi
 
-set -x
+for platform in ${platforms/,/ }
-docker run --rm \
+do
-  --mount type=bind,src=${PROJECT_ROOT},dst=${CORSO_BUILD_CONTAINER}          \
+  IFS='/' read -r -a platform_split <<< "${platform}"
-  --mount type=bind,src=${CORSO_BUILD_TMP_CACHE},dst=${CORSO_BUILD_TMP_CACHE} \
+  GOOS=${platform_split[0]}
-  --mount type=bind,src=${CORSO_BUILD_TMP_MOD},dst=${CORSO_BUILD_PKG_MOD}     \
+  GOARCH=${platform_split[1]}
-  --workdir ${CORSO_BUILD_CONTAINER_SRC}                                      \
-  --env GOMODCACHE=${CORSO_MOD_CACHE}                                         \
-  --env GOCACHE=${CORSO_CACHE}                                                \
-  --env GOOS=${GOOS}                                                          \
-  --env GOARCH=${GOARCH}                                                      \
-  --entrypoint /usr/local/go/bin/go                                           \
-  golang:${GOVER}                                                             \
-  build ${CORSO_BUILD_ARGS}
-set +x
 
-mkdir -p ${PROJECT_ROOT}/bin
+  echo "-----"
-mv ${PROJECT_ROOT}/src/corso ${PROJECT_ROOT}/bin/corso
+  echo "building corso binary for ${GOOS}/${GOARCH}"
+  echo "-----"
 
-echo "-----"
+  set -x
-echo "created binary image in ${PROJECT_ROOT}/bin/corso"
+  docker run --rm \
+    --mount type=bind,src=${PROJECT_ROOT},dst=${CORSO_BUILD_CONTAINER}          \
+    --mount type=bind,src=${CORSO_BUILD_TMP_CACHE},dst=${CORSO_BUILD_TMP_CACHE} \
+    --mount type=bind,src=${CORSO_BUILD_TMP_MOD},dst=${CORSO_BUILD_PKG_MOD}     \
+    --workdir ${CORSO_BUILD_CONTAINER_SRC}                                      \
+    --env GOMODCACHE=${CORSO_MOD_CACHE}                                         \
+    --env GOCACHE=${CORSO_CACHE}                                                \
+    --env GOOS=${GOOS}                                                          \
+    --env GOARCH=${GOARCH}                                                      \
+    --entrypoint /usr/local/go/bin/go                                           \
+    golang:${GOVER}                                                             \
+    build ${CORSO_BUILD_ARGS}
+  set +x
+
+  mkdir -p ${PROJECT_ROOT}/bin/${GOOS}-${GOARCH}
+  mv ${PROJECT_ROOT}/src/corso ${PROJECT_ROOT}/bin/${GOOS}-${GOARCH}/corso
+
+  echo "-----"
+  echo "created binary image in ${PROJECT_ROOT}/bin/${GOOS}-${GOARCH}/corso"
+  echo "-----"
+done

build/multiplatform-binary.sh

@@ -0,0 +1,56 @@
+#!/bin/bash
+
+set -e
+
+SCRIPT_ROOT=$(dirname $(readlink -f $0))
+PROJECT_ROOT=$(dirname ${SCRIPT_ROOT})
+
+platforms=
+GOVER=1.18
+GOOS=linux
+GOARCH=amd64
+
+while [ "$#" -gt 0 ]
+do
+  case "$1" in
+  --platforms)
+    platforms=$2
+    shift
+    ;;
+  esac
+  shift
+done
+
+CORSO_BUILD_ARGS="$@"
+
+if [ -z "$platforms" ]; then
+  platforms="${GOOS}/${GOARCH}"
+fi
+
+for platform in ${platforms/,/ }
+do
+  IFS='/' read -r -a platform_split <<< "${platform}"
+  GOOS=${platform_split[0]}
+  GOARCH=${platform_split[1]}
+
+  echo "-----"
+  echo "building corso binary for ${GOOS}/${GOARCH}"
+  echo "-----"
+
+  OS_ARCH_DIR=${PROJECT_ROOT}/bin/${GOOS}-${GOARCH}
+
+  set -x
+
+  mkdir -p ${OS_ARCH_DIR}
+
+  cd ${PROJECT_ROOT}/src; \
+  GOOS=${GOOS}            \
+  GOARCH=${GOARCH}        \
+  go build -o ${OS_ARCH_DIR} "$CORSO_BUILD_ARGS"
+
+  set +x
+
+  echo "-----"
+  echo "created binary ${PROJECT_ROOT}/bin/${GOOS}-${GOARCH}/corso"
+  echo "-----"
+done

docker/docker-bake.hcl

@@ -1,12 +0,0 @@
-// docker-bake.hcl
-target "docker-metadata-action" {}
-
-target "build" {
-  inherits = ["docker-metadata-action"]
-  context = "./"
-  dockerfile = "Dockerfile"
-  platforms = [
-    "linux/amd64",
-    "linux/arm64",
-  ]
-}