Merge 197d76de14 into 3d9490d335

.github/workflows/main.yml

@@ -16,9 +16,34 @@ permissions:
   contents: read
   packages: write
 
-# build components using a job matrix, so they can run in parallel
+
 jobs:
+  # compile buildah v1.38.0, so we can have a more up-to-date version than ubuntu 24.04 has in its repos
+  buildah:
+    runs-on: ubuntu-24.04
+    steps:
+      - name: Install buildah dependencies
+        run: |
+          sudo apt update
+          sudo apt install -y bats btrfs-progs git go-md2man golang libapparmor-dev libglib2.0-dev libgpgme11-dev libseccomp-dev libselinux1-dev make runc skopeo libbtrfs-dev
+      - name: Pull buildah sources
+        run: |
+          git clone https://github.com/containers/buildah
+          cd buildah
+          git checkout v1.38.0
+      - name: Build buildah
+        run: |
+          cd buildah
+          make
+          sudo make install
+      - name: Save buildah build artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: buildah-bin
+          path: /usr/local/bin/buildah
+  # build components using a job matrix, so they can run in parallel
   build:
+    needs: buildah
     runs-on: ubuntu-24.04
     strategy:
       fail-fast: false
@@ -40,8 +65,22 @@ jobs:
           - thrimshim
           - zulip_bots
     steps:
+      - name: Fetch buildah build artifact
+        uses: actions/download-artifact@v4
+        with:
+          name: buildah-bin
+          path: /usr/local/bin/buildah
+      - name: Ensure buildah binary has +x permissions set
+        run: |
+          sudo chmod +x /usr/local/bin/buildah
+      - name: Check buildah version
+        run: |
+          ls -l /usr/local/bin/buildah
+          echo $PATH
+          which buildah
+          buildah version
       - name: Check out repo
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
         with:
           submodules: recursive
           # By default, for PR builds, Actions will check out a merge commit between the actual
@@ -51,8 +90,14 @@ jobs:
           # This option makes it actually check out the PR's commit instead.
           ref: ${{ github.event.pull_request.head.sha }}
 
-      - name: Log into ghcr.io
-        uses: docker/login-action@v1
+      - name: Install QEMU
+        # qemu-user-static is used by buildah to do multiplatform builds
+        run: |
+          sudo apt update
+          sudo apt install -y qemu-user-static
+
+      - name: Log in to ghcr.io
+        uses: docker/login-action@v3
         with:
           registry: ghcr.io
           username: ${{ github.actor }}
@@ -62,8 +107,9 @@ jobs:
         # always push
         # if not a pull request and a push to master, also push "latest" tag
         # try to cache from previous build and then build component using build script
+        # set the multiarch flag so we also build arm64 images and not just amd64
         run: |
-          export PUSH=true CACHE=true
+          export PUSH=true CACHE=true MULTIPLATFORM=true
           # Only push latest when pushing to master
           if [ "$GITHUB_EVENT_NAME" != "pull_request" ] && [ "$GITHUB_REF" == "refs/heads/master" ]; then export PUSH=latest; fi
           # PRs from forked repositories can't have package write permissions, so use cache in readonly mode in those cases.

build

@@ -55,7 +55,7 @@ case "${CACHE:-}" in
 		BUILD_CMD="docker"
 		;;
 	readonly)
-		CACHE_ARGS+=("--cache-from" "$CACHE_NAME")
+		CACHE_ARGS+=("--layers" "--cache-from" "$CACHE_NAME")
 		BUILD_CMD="buildah"
 		;;
 	*)
@@ -63,24 +63,54 @@ case "${CACHE:-}" in
 		BUILD_CMD="buildah"
 		;;
 esac
-		
+
+# check for MULTIPLATFORM flag. error out if we're not using buildah for now. docker buildx should be able to do it too, but needs testing
+if [ -n "$MULTIPLATFORM" ]; then
+	if [ "$BUILD_CMD" == "docker" ]; then
+		echo "Multiplatform builds are not yet supported with docker! Use buildah instead."
+		exit 1
+	fi
+fi
+
 for component in "${COMPONENTS[@]}"; do
 	echo "Building image for $component"
 	latest="$BASE/wubloader-$component:latest"
 	specific="$BASE/wubloader-$component:$TAG"
-	"$BUILD_CMD" build \
-		-f "$component/Dockerfile" \
-		-t "$latest" \
-		-t "$specific" \
-		"${CACHE_ARGS[@]}" \
-		.
-	echo "Built image wubloader-$component:$TAG"
-	if [ -n "$PUSH" ]; then
-		echo "Pushing tag $specific"
-		"$BUILD_CMD" push "$specific"
-	fi
-	if [ "$PUSH" == "latest" ]; then
-		echo "Pushing tag $latest"
-		"$BUILD_CMD" push "$latest"
+	if [ -n "$MULTIPLATFORM" ]; then
+		"$BUILD_CMD" build \
+			-f "$component/Dockerfile" \
+			--manifest "$specific" \
+			--manifest "$latest" \
+			--jobs 2 \
+			--platform=linux/amd64,linux/arm64 \
+			"${CACHE_ARGS[@]}" \
+			.
+	else
+		"$BUILD_CMD" build \
+			-f "$component/Dockerfile" \
+			-t "$latest" \
+			-t "$specific" \
+			"${CACHE_ARGS[@]}" \
+			.
 	fi
+	echo "Built image wubloader-$component:$TAG"
+    if [ -n "$MULTIPLATFORM" ]; then
+	    if [ -n "$PUSH" ]; then
+		    echo "Pushing tag $specific"
+		    "$BUILD_CMD" manifest push "$specific"
+	    fi
+	    if [ "$PUSH" == "latest" ]; then
+		    echo "Pushing tag $latest"
+		    "$BUILD_CMD" manifest push "$latest"
+	    fi
+    else
+	    if [ -n "$PUSH" ]; then
+		    echo "Pushing tag $specific"
+		    "$BUILD_CMD" push "$specific"
+	    fi
+	    if [ "$PUSH" == "latest" ]; then
+		    echo "Pushing tag $latest"
+		    "$BUILD_CMD" push "$latest"
+	    fi
+    fi
 done

buscribe/Dockerfile

@@ -1,4 +1,4 @@
-FROM debian:11
+FROM debian:12
 
 RUN apt update &&\
     apt install -y python3 libpq-dev python3-pip curl unzip ffmpeg
@@ -14,6 +14,8 @@ RUN cd /tmp \
 	&& unzip vosk-model-spk-0.4.zip -d /usr/share/buscribe \
 	&& rm *.zip
 
+RUN rm /usr/lib/python3.11/EXTERNALLY-MANAGED
+
 COPY common /tmp/common
 RUN pip install /tmp/common && rm -r /tmp/common