diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7714e3023..a37695cce 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,6 +1,6 @@ name: Build/Release Corso on: - workflow_dispatch: # TODO(meain): post-merge: verify manual dispatch + workflow_dispatch: pull_request: branches: [ main ] push: @@ -238,7 +238,7 @@ jobs: run: echo "::set-output name=version::$(git describe --exact-match --tags $(git rev-parse HEAD) 2>/dev/null || echo unreleased)-$(git rev-parse --short HEAD)" - name: Run GoReleaser - uses: goreleaser/goreleaser-action@v3 # TODO(meain): make sure release builds work + uses: goreleaser/goreleaser-action@v3 with: version: latest args: release --rm-dist --timeout 500m @@ -285,7 +285,6 @@ jobs: npm ci CORSO_DOCS_BASEURL="/preview/" npm run build # TODO: update base url once finalized - # TODO(meain): post-merge: validate push to prod env - name: Push docs run: | echo "$DOCS_BUCKET" | base64 @@ -309,20 +308,10 @@ jobs: steps: - uses: actions/checkout@v3 - - name: Build Corso Binaries - run: | - export CORSO_BUILD_LDFLAGS="-X 'github.com/alcionai/corso/src/internal/events.RudderStackWriteKey=${{ secrets.RUDDERSTACK_CORSO_WRITE_KEY }}' \ - -X 'github.com/alcionai/corso/src/internal/events.RudderStackDataPlaneURL=${{ secrets.RUDDERSTACK_CORSO_DATA_PLANE_URL }}' \ - -X 'github.com/alcionai/corso/src/cli.version=$(git describe --exact-match --tags $(git rev-parse HEAD) 2>/dev/null || echo unreleased)-$(git rev-parse --short HEAD)'" - ./build.sh --platforms ${{ env.PLATFORMS }} - - # apparently everyone uses this step + # Setup buildx - name: Set up QEMU uses: docker/setup-qemu-action@v2 - - # setup Docker build action - name: Set up Docker Buildx - id: buildx uses: docker/setup-buildx-action@v2 # retrieve credentials for ghcr.io @@ -337,7 +326,7 @@ jobs: id: meta uses: docker/metadata-action@v4 with: - images: ${{ env.imageName }} # TODO(meain): post-merge: validate push with tag + images: ${{ env.imageName }} tags: | type=ref,event=tag type=sha,format=short,prefix= @@ -352,6 +341,8 @@ jobs: push: true tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} + build-args: | + CORSO_BUILD_LDFLAGS=-X 'github.com/alcionai/corso/src/internal/events.RudderStackWriteKey=${{ secrets.RUDDERSTACK_CORSO_WRITE_KEY }}' -X 'github.com/alcionai/corso/src/internal/events.RudderStackDataPlaneURL=${{ secrets.RUDDERSTACK_CORSO_DATA_PLANE_URL }}' -X 'github.com/alcionai/corso/src/cli.version=$(git describe --exact-match --tags $(git rev-parse HEAD) 2>/dev/null || echo unreleased)-$(git rev-parse --short HEAD)' # use the github cache cache-from: type=gha cache-to: type=gha,mode=max \ No newline at end of file diff --git a/build/Dockerfile b/build/Dockerfile index 652f65d5f..8ce5809e9 100644 --- a/build/Dockerfile +++ b/build/Dockerfile @@ -1,34 +1,31 @@ -# syntax=docker/dockerfile:1 +FROM golang:1.18-alpine as builder -# This dockerfile is configured to be run by the /corso/build/build-container.sh -# script. Using docker to build this file directly will fail. +WORKDIR /go/src/app +COPY src . + +ARG CORSO_BUILD_LDFLAGS="" # ldflags +RUN go build -o corso -ldflags "$CORSO_BUILD_LDFLAGS" FROM alpine:3.16 -ENV CORSO_HOME /app/corso +LABEL org.opencontainers.image.title="Corso" +LABEL org.opencontainers.image.description="Free, Secure, and Open-Source Backup for Microsoft 365" +LABEL org.opencontainers.image.url="https://github.com/alcionai/corso" +LABEL org.opencontainers.image.source="https://github.com/alcionai/corso" +LABEL org.opencontainers.image.vendor="Alcion, Inc." -# Add a well-know corso user to use by default -RUN addgroup -g 1001 corso && \ - adduser --shell /sbin/nologin --disabled-password \ - --home $CORSO_HOME --uid 1001 --ingroup corso corso +COPY --from=builder /go/src/app/corso /corso -# Update to latest cert bundle in case there are changes -RUN apk update && apk add ca-certificates && rm -rf /var/cache/apk/* +# Pull tls certs directly from latest upstream image +COPY --from=alpine:latest /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ -# Default locations for key Kopia config as set in https://github.com/kopia/kopia/blob/master/tools/docker/Dockerfile -ENV KOPIA_CONFIG_PATH=$CORSO_HOME/kopia/config/repository.config \ +ENV CORSO_HOME=/app/corso +ENV CORSO_CONFIG_DIR=$CORSO_HOME \ + KOPIA_CONFIG_PATH=$CORSO_HOME/kopia/config/repository.config \ KOPIA_LOG_DIR=$CORSO_HOME/kopia/logs \ KOPIA_CACHE_DIRECTORY=$CORSO_HOME/kopia/cache \ RCLONE_CONFIG=$CORSO_HOME/kopia/rclone/rclone.conf \ KOPIA_PERSIST_CREDENTIALS_ON_CONNECT=false \ KOPIA_CHECK_FOR_UPDATES=false -WORKDIR / - -ARG TARGETOS -ARG TARGETARCH -COPY ./bin/${TARGETOS}-${TARGETARCH}/corso ./ - -USER corso - -ENTRYPOINT ["/corso"] +ENTRYPOINT ["/corso"] \ No newline at end of file diff --git a/build/build-container.sh b/build/build-container.sh deleted file mode 100755 index 8b9144ea1..000000000 --- a/build/build-container.sh +++ /dev/null @@ -1,100 +0,0 @@ -#!/bin/sh - -set -e - -usage() { - echo " " - echo "-----" - echo "Builds a Corso docker container image." - echo " " - echo "-----" - echo "Flags" - echo " -h|--help Help" - echo " -a|--arch Set the architecture to the specified value (default: amd64)" - echo " -l|--local Build the corso binary on your local system, rather than a go image" - echo " -p|--prefix Prefixes the image name." - echo " -s|--suffix Suffixes the version." - echo " " - echo "-----" - echo "Example Usage:" - echo " ./build/build-container.sh" - echo " ./build/build-container.sh --arch arm64" - echo " ./build/build-container.sh --arch arm64 --prefix ghcr.io --suffix nightly" - echo " " - exit 0 -} - -SCRIPT_ROOT=$(dirname $(readlink -f $0)) -PROJECT_ROOT=$(dirname ${SCRIPT_ROOT}) - -OS=linux -ARCH=amd64 -IMAGE_NAME_PREFIX= -IMAGE_TAG_SUFFIX= -LOCAL= - -while [ "$#" -gt 0 ] -do - case "$1" in - -h|--help) - usage - exit 0 - ;; - -a|--arch) - ARCH=$2 - shift - ;; - -l|--local) - LOCAL=1 - ;; - -p|--prefix) - IMAGE_NAME_PREFIX=$2 - shift - ;; - -s|--suffix) - IMAGE_TAG_SUFFIX=$2 - shift - ;; - -*) - echo "Invalid flag '$1'. Use -h|--help to see the valid options" >&2 - return 1 - ;; - *) - echo "Invalid arg '$1'. Use -h|--help to see the valid options" >&2 - return 1 - ;; - esac - shift -done - -TARGETPLATFORM=${OS}/${ARCH} - -IMAGE_TAG=${OS}-${ARCH} -if [ ! -z "${IMAGE_TAG_SUFFIX}" ]; then - IMAGE_TAG=${IMAGE_TAG}-${IMAGE_TAG_SUFFIX} -fi - -IMAGE_NAME=alcionai/corso:${IMAGE_TAG} -if [ ! -z "${IMAGE_NAME_PREFIX}" ]; then - IMAGE_NAME=${IMAGE_NAME_PREFIX}/${IMAGE_NAME} -fi - -if [ -z "$LOCAL" ]; then - ${SCRIPT_ROOT}/build.sh --platforms "${TARGETPLATFORM}" -else - ${SCRIPT_ROOT}/multiplatform-binary.sh --platforms "${TARGETPLATFORM}" -fi - -echo "-----" -echo "building corso container ${IMAGE_NAME}" -echo "-----" - -set -x -docker buildx build --tag ${IMAGE_NAME} \ - --platform ${TARGETPLATFORM} \ - --file ${PROJECT_ROOT}/build/Dockerfile \ - ${PROJECT_ROOT} -set +x - -echo "-----" -echo "container built successfully" diff --git a/build/build.sh b/build/build.sh index aff255902..208538b93 100755 --- a/build/build.sh +++ b/build/build.sh @@ -2,74 +2,80 @@ set -e -SCRIPT_ROOT=$(dirname $(readlink -f $0)) -PROJECT_ROOT=$(dirname ${SCRIPT_ROOT}) +ROOT=$(dirname $(dirname $(readlink -f $0))) +GOVER=1.18 # go version +CORSO_BUILD_CACHE="/tmp/.corsobuild" # shared persistent cache -CORSO_BUILD_CONTAINER=/go/src/github.com/alcionai/corso -CORSO_BUILD_CONTAINER_SRC=${CORSO_BUILD_CONTAINER}/src -CORSO_BUILD_PKG_MOD=/go/pkg/mod -CORSO_BUILD_TMP=/tmp/.corsobuild -CORSO_BUILD_TMP_CACHE=${CORSO_BUILD_TMP}/cache -CORSO_BUILD_TMP_MOD=${CORSO_BUILD_TMP}/mod -CORSO_CACHE=${CORSO_BUILD_TMP_CACHE} -CORSO_MOD_CACHE=${CORSO_BUILD_PKG_MOD}/cache +# Figure out os and architecture +case "$(uname -m)" in +x86_64) GOARCH="amd64" ;; +aarch64) GOARCH="arm64" ;; +arm) GOARCH="arm" ;; +i386) GOARCH="386" ;; +*) echo "Unknown architecture" && exit 0 ;; +esac +case "$(uname)" in +Linux) GOOS="linux" ;; +Darwin) GOOS="darwin" ;; # TODO: verify this +*) echo "Unknown OS" && exit 0 ;; +esac -CORSO_BUILD_ARGS='' +PLATFORMS="$GOOS/$GOARCH" # default platform +TAG="alcionai/corso" # default image tag -platforms= -GOVER=1.18 -GOOS=linux -GOARCH=amd64 +usage() { + echo "Usage: $(basename $0) [--platforms ...] [--tag ...]" + echo "" + echo "OPTIONS" + echo " -p|--platforms Platforms to build for (default: $PLATFORMS)" + echo " Specify multiple platforms using ',' (eg: linux/amd64,darwin/arm)" + echo " -t|--tag Tag for container image (default: $TAG)" +} -while [ "$#" -gt 0 ] -do - case "$1" in - --platforms) - platforms=$2 - shift - ;; - esac - shift +MODE="binary" +case "$1" in +binary) MODE="binary" && shift ;; +image) MODE="image" && shift ;; +-h | --help) usage && exit 0 ;; +*) usage && exit 1 ;; +esac + +while [ "$#" -gt 0 ]; do + case "$1" in + -p | --platforms) PLATFORMS="$2" && shift ;; + -t | --tag) TAG="$2" && shift ;; + *) echo "Invalid argument $1" && usage && exit 1 ;; + esac + shift done -# temporary directory for caching go build -mkdir -p ${CORSO_BUILD_TMP_CACHE} -# temporary directory for caching go modules (needed for fast cross-platform build) -mkdir -p ${CORSO_BUILD_TMP_MOD} +if [ "$MODE" == "binary" ]; then + mkdir -p ${CORSO_BUILD_CACHE} # prep env + for platform in ${PLATFORMS/,/ }; do + IFS='/' read -r -a platform_split <<<"$platform" + GOOS=${platform_split[0]} + GOARCH=${platform_split[1]} -if [ -z "$platforms" ]; then - platforms="${GOOS}/${GOARCH}" + printf "Building for %s...\r" "$platform" + docker run --rm \ + --mount type=bind,src=${ROOT},dst="/app" \ + --mount type=bind,src=${CORSO_BUILD_CACHE},dst=${CORSO_BUILD_CACHE} \ + --env GOMODCACHE=${CORSO_BUILD_CACHE}/mod --env GOCACHE=${CORSO_BUILD_CACHE}/cache \ + --env GOOS=${GOOS} --env GOARCH=${GOARCH} \ + --workdir "/app/src" \ + golang:${GOVER} \ + go build -o corso -ldflags "${CORSO_BUILD_LDFLAGS}" + + mkdir -p ${ROOT}/bin/${GOOS}-${GOARCH} + mv ${ROOT}/src/corso ${ROOT}/bin/${GOOS}-${GOARCH}/corso + echo Corso $platform binary available in ${ROOT}/bin/${GOOS}-${GOARCH}/corso + done +else + echo Building "$TAG" image for "$PLATFORMS" + docker buildx build --tag ${TAG} \ + --platform ${PLATFORMS} \ + --file ${ROOT}/build/Dockerfile \ + --build-arg CORSO_BUILD_LDFLAGS="$CORSO_BUILD_LDFLAGS" \ + --load ${ROOT} + echo Built container image "$TAG" fi - -for platform in ${platforms/,/ } -do - IFS='/' read -r -a platform_split <<< "${platform}" - GOOS=${platform_split[0]} - GOARCH=${platform_split[1]} - - echo "-----" - echo "building corso binary for ${GOOS}/${GOARCH}" - echo "-----" - - set -x - docker run --rm \ - --mount type=bind,src=${PROJECT_ROOT},dst=${CORSO_BUILD_CONTAINER} \ - --mount type=bind,src=${CORSO_BUILD_TMP_CACHE},dst=${CORSO_BUILD_TMP_CACHE} \ - --mount type=bind,src=${CORSO_BUILD_TMP_MOD},dst=${CORSO_BUILD_PKG_MOD} \ - --workdir ${CORSO_BUILD_CONTAINER_SRC} \ - --env GOMODCACHE=${CORSO_MOD_CACHE} \ - --env GOCACHE=${CORSO_CACHE} \ - --env GOOS=${GOOS} \ - --env GOARCH=${GOARCH} \ - --entrypoint /usr/local/go/bin/go \ - golang:${GOVER} \ - build -o corso ${CORSO_BUILD_ARGS} -ldflags "${CORSO_BUILD_LDFLAGS}" - set +x - - mkdir -p ${PROJECT_ROOT}/bin/${GOOS}-${GOARCH} - mv ${PROJECT_ROOT}/src/corso ${PROJECT_ROOT}/bin/${GOOS}-${GOARCH}/corso - - echo "-----" - echo "created binary image in ${PROJECT_ROOT}/bin/${GOOS}-${GOARCH}/corso" - echo "-----" -done \ No newline at end of file diff --git a/build/multiplatform-binary.sh b/build/multiplatform-binary.sh deleted file mode 100755 index ba65765a8..000000000 --- a/build/multiplatform-binary.sh +++ /dev/null @@ -1,56 +0,0 @@ -#!/bin/bash - -set -e - -SCRIPT_ROOT=$(dirname $(readlink -f $0)) -PROJECT_ROOT=$(dirname ${SCRIPT_ROOT}) - -platforms= -GOVER=1.18 -GOOS=linux -GOARCH=amd64 - -while [ "$#" -gt 0 ] -do - case "$1" in - --platforms) - platforms=$2 - shift - ;; - esac - shift -done - -CORSO_BUILD_ARGS="$@" - -if [ -z "$platforms" ]; then - platforms="${GOOS}/${GOARCH}" -fi - -for platform in ${platforms/,/ } -do - IFS='/' read -r -a platform_split <<< "${platform}" - GOOS=${platform_split[0]} - GOARCH=${platform_split[1]} - - echo "-----" - echo "building corso binary for ${GOOS}/${GOARCH}" - echo "-----" - - OS_ARCH_DIR=${PROJECT_ROOT}/bin/${GOOS}-${GOARCH} - - set -x - - mkdir -p ${OS_ARCH_DIR} - - cd ${PROJECT_ROOT}/src; \ - GOOS=${GOOS} \ - GOARCH=${GOARCH} \ - go build -o ${OS_ARCH_DIR}/corso $CORSO_BUILD_ARGS - - set +x - - echo "-----" - echo "created binary ${PROJECT_ROOT}/bin/${GOOS}-${GOARCH}/corso" - echo "-----" -done \ No newline at end of file diff --git a/docs/docs/developers/build.md b/docs/docs/developers/build.md index 26868a2cf..8e1ee3f6f 100644 --- a/docs/docs/developers/build.md +++ b/docs/docs/developers/build.md @@ -13,7 +13,7 @@ go build -o corso ``` :::info -If you dong have Go available, you can find installation instructions [here](https://go.dev/doc/install) +If you don't have Go available, you can find installation instructions [here](https://go.dev/doc/install) ::: This will generate a binary named `corso` in the directory where you run the build. @@ -26,16 +26,17 @@ For convenience, the Corso build tooling is containerized. To take advantage, yo To build Corso via docker, use the following command from the root of your repo: ```bash -./build/build.sh +./build/build.sh binary ``` -You can pass in all the architectures/platforms you would like to -build it for using the `--platforms` flag as a comma separated -list. For example, if you would like to build `amd64` and `arm64` -versions for Linux, you can run the following command: +By default, we will build for your current platform. You can pass in +all the architectures/platforms you would like to build it for using +the `--platforms` flag as a comma separated list. For example, if you +would like to build `amd64` and `arm64` versions for Linux, you can +run the following command: ```bash -./build/build.sh --platforms linux/amd64,linux/arm64 +./build/build.sh binary --platforms linux/amd64,linux/arm64 ``` Once built, the resulting binaries will be available in `/bin` for all the different platforms you specified. @@ -46,18 +47,29 @@ If you prefer to build Corso as a container image, use the following command ins ```bash # Use --help to see all available options -./build/build-container.sh +./build/build.sh image ``` -Below are the main customization flags that you can set when building a container image: +:::note +`Dockerfile` used to build the image is available at [`build/Dockerfile`](https://github.com/alcionai/corso/blob/main/build/Dockerfile) +::: -- `-a|--arch`: Set the architecture to the specified value. (default: amd64) -- `-l|--local`: Build the corso binary on your local system, rather than a go image. -- `-p|--prefix`: Prefix for the image name. -- `-s|--suffix`: Suffix for the version. +Similar to binaries, we build your a container image for your current +platform by default, but you can change it by explicitly passing in +the platforms that you would like to build for. +In addition, you can optionally pass the tag that you would like to +apply for the image using `--tag` option. -For example, you can use the following command to create a `arm64` image with prefix of `ghcr.io` and the tag as `nightly`. +For example, you can use the following command to create a `arm64` +image with the tag `gcr.io/alcionai/corso:latest`, you can run: ```bash -./build/build-container.sh --arch arm64 --prefix ghcr.io --suffix nightly +./build/build.sh image --platforms linux/arm64 --tag gcr.io/alcionai/corso:latest ``` + +:::info +If you run into any issues with building cross platform images, make +sure to follow the instructions on [Docker +docs](https://docs.docker.com/build/building/multi-platform/) to setup +the build environment for Multi-platform images. +:::