diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index f56e3c6..6b4254a 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -16,9 +16,34 @@ permissions: contents: read packages: write -# build components using a job matrix, so they can run in parallel + jobs: + # compile buildah v1.38.0, so we can have a more up-to-date version than ubuntu 24.04 has in its repos + buildah: + runs-on: ubuntu-24.04 + steps: + - name: Install buildah dependencies + run: | + sudo apt update + sudo apt install -y bats btrfs-progs git go-md2man golang libapparmor-dev libglib2.0-dev libgpgme11-dev libseccomp-dev libselinux1-dev make runc skopeo libbtrfs-dev + - name: Pull buildah sources + run: | + git clone https://github.com/containers/buildah + cd buildah + git checkout v1.38.0 + - name: Build buildah + run: | + cd buildah + make + sudo make install + - name: Save buildah build artifact + uses: actions/upload-artifact@v4 + with: + name: buildah-bin + path: /usr/local/bin/buildah + # build components using a job matrix, so they can run in parallel build: + needs: buildah runs-on: ubuntu-24.04 strategy: fail-fast: false @@ -40,8 +65,22 @@ jobs: - thrimshim - zulip_bots steps: + - name: Fetch buildah build artifact + uses: actions/download-artifact@v4 + with: + name: buildah-bin + path: /usr/local/bin/buildah + - name: Ensure buildah binary has +x permissions set + run: | + sudo chmod +x /usr/local/bin/buildah + - name: Check buildah version + run: | + ls -l /usr/local/bin/buildah + echo $PATH + which buildah + buildah version - name: Check out repo - uses: actions/checkout@v2 + uses: actions/checkout@v4 with: submodules: recursive # By default, for PR builds, Actions will check out a merge commit between the actual @@ -51,8 +90,14 @@ jobs: # This option makes it actually check out the PR's commit instead. ref: ${{ github.event.pull_request.head.sha }} - - name: Log into ghcr.io - uses: docker/login-action@v1 + - name: Install QEMU + # qemu-user-static is used by buildah to do multiplatform builds + run: | + sudo apt update + sudo apt install -y qemu-user-static + + - name: Log in to ghcr.io + uses: docker/login-action@v3 with: registry: ghcr.io username: ${{ github.actor }} @@ -62,8 +107,9 @@ jobs: # always push # if not a pull request and a push to master, also push "latest" tag # try to cache from previous build and then build component using build script + # set the multiarch flag so we also build arm64 images and not just amd64 run: | - export PUSH=true CACHE=true + export PUSH=true CACHE=true MULTIPLATFORM=true # Only push latest when pushing to master if [ "$GITHUB_EVENT_NAME" != "pull_request" ] && [ "$GITHUB_REF" == "refs/heads/master" ]; then export PUSH=latest; fi # PRs from forked repositories can't have package write permissions, so use cache in readonly mode in those cases. diff --git a/build b/build index d8a1fa6..6cefd40 100755 --- a/build +++ b/build @@ -55,7 +55,7 @@ case "${CACHE:-}" in BUILD_CMD="docker" ;; readonly) - CACHE_ARGS+=("--cache-from" "$CACHE_NAME") + CACHE_ARGS+=("--layers" "--cache-from" "$CACHE_NAME") BUILD_CMD="buildah" ;; *) @@ -63,24 +63,54 @@ case "${CACHE:-}" in BUILD_CMD="buildah" ;; esac - + +# check for MULTIPLATFORM flag. error out if we're not using buildah for now. docker buildx should be able to do it too, but needs testing +if [ -n "$MULTIPLATFORM" ]; then + if [ "$BUILD_CMD" == "docker" ]; then + echo "Multiplatform builds are not yet supported with docker! Use buildah instead." + exit 1 + fi +fi + for component in "${COMPONENTS[@]}"; do echo "Building image for $component" latest="$BASE/wubloader-$component:latest" specific="$BASE/wubloader-$component:$TAG" - "$BUILD_CMD" build \ - -f "$component/Dockerfile" \ - -t "$latest" \ - -t "$specific" \ - "${CACHE_ARGS[@]}" \ - . - echo "Built image wubloader-$component:$TAG" - if [ -n "$PUSH" ]; then - echo "Pushing tag $specific" - "$BUILD_CMD" push "$specific" - fi - if [ "$PUSH" == "latest" ]; then - echo "Pushing tag $latest" - "$BUILD_CMD" push "$latest" + if [ -n "$MULTIPLATFORM" ]; then + "$BUILD_CMD" build \ + -f "$component/Dockerfile" \ + --manifest "$specific" \ + --manifest "$latest" \ + --jobs 2 \ + --platform=linux/amd64,linux/arm64 \ + "${CACHE_ARGS[@]}" \ + . + else + "$BUILD_CMD" build \ + -f "$component/Dockerfile" \ + -t "$latest" \ + -t "$specific" \ + "${CACHE_ARGS[@]}" \ + . fi + echo "Built image wubloader-$component:$TAG" + if [ -n "$MULTIPLATFORM" ]; then + if [ -n "$PUSH" ]; then + echo "Pushing tag $specific" + "$BUILD_CMD" manifest push "$specific" + fi + if [ "$PUSH" == "latest" ]; then + echo "Pushing tag $latest" + "$BUILD_CMD" manifest push "$latest" + fi + else + if [ -n "$PUSH" ]; then + echo "Pushing tag $specific" + "$BUILD_CMD" push "$specific" + fi + if [ "$PUSH" == "latest" ]; then + echo "Pushing tag $latest" + "$BUILD_CMD" push "$latest" + fi + fi done diff --git a/buscribe/Dockerfile b/buscribe/Dockerfile index 24e8eba..45496ce 100644 --- a/buscribe/Dockerfile +++ b/buscribe/Dockerfile @@ -1,4 +1,4 @@ -FROM debian:11 +FROM debian:12 RUN apt update &&\ apt install -y python3 libpq-dev python3-pip curl unzip ffmpeg @@ -14,6 +14,8 @@ RUN cd /tmp \ && unzip vosk-model-spk-0.4.zip -d /usr/share/buscribe \ && rm *.zip +RUN rm /usr/lib/python3.11/EXTERNALLY-MANAGED + COPY common /tmp/common RUN pip install /tmp/common && rm -r /tmp/common