From 4c943a07376b1527fd90a3136ce144a608cfa97b Mon Sep 17 00:00:00 2001
From: Hubbe <HubbeKing@users.noreply.github.com>
Date: Fri, 8 Nov 2024 09:01:26 +0200
Subject: [PATCH 01/28] Install qemu in actions workflow

---
 .github/workflows/main.yml | 13 ++++++++++---
 1 file changed, 10 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index f56e3c6..b6c83c7 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -41,7 +41,7 @@ jobs:
           - zulip_bots
     steps:
       - name: Check out repo
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
         with:
           submodules: recursive
           # By default, for PR builds, Actions will check out a merge commit between the actual
@@ -51,8 +51,14 @@ jobs:
           # This option makes it actually check out the PR's commit instead.
           ref: ${{ github.event.pull_request.head.sha }}
 
+      - name: Install QEMU
+        # qemu-user-static is used by buildah to do multiplatform builds
+        run: |
+          sudo apt update
+          sudo apt install -y qemu-user-static
+
       - name: Log into ghcr.io
-        uses: docker/login-action@v1
+        uses: docker/login-action@v3
         with:
           registry: ghcr.io
           username: ${{ github.actor }}
@@ -62,8 +68,9 @@ jobs:
         # always push
         # if not a pull request and a push to master, also push "latest" tag
         # try to cache from previous build and then build component using build script
+        # set the multiarch flag so we also build arm64 images and not just amd64
         run: |
-          export PUSH=true CACHE=true
+          export PUSH=true CACHE=true MULTIPLATFORM=true
           # Only push latest when pushing to master
           if [ "$GITHUB_EVENT_NAME" != "pull_request" ] && [ "$GITHUB_REF" == "refs/heads/master" ]; then export PUSH=latest; fi
           # PRs from forked repositories can't have package write permissions, so use cache in readonly mode in those cases.

From cc7d1ba9318a8aa697fc1873fcb1db8b363abd49 Mon Sep 17 00:00:00 2001
From: Hubbe <HubbeKing@users.noreply.github.com>
Date: Fri, 8 Nov 2024 09:13:53 +0200
Subject: [PATCH 02/28] Add buildah multiarch build commands to build script

---
 build | 50 +++++++++++++++++++++++++++++++++++++++++++-------
 1 file changed, 43 insertions(+), 7 deletions(-)

diff --git a/build b/build
index 806149a..aef3912 100755
--- a/build
+++ b/build
@@ -62,24 +62,60 @@ case "${CACHE:-}" in
 		BUILD_CMD="buildah"
 		;;
 esac
+
+# check for MULTIPLATFORM flag. error out if we're not using buildah for now. docker buildx should be able to do it too, but needs testing
+if [ -n "$MULTIPLATFORM" ]; then
+	if [ "$BUILD_CMD" == "docker" ]; then
+		echo "Multiplatform builds are not yet supported with docker! Use buildah instead."
+		exit 1
+	fi
+fi
 		
 for component in "${COMPONENTS[@]}"; do
 	echo "Building image for $component"
+	if [ -n "$MULTIPLATFORM" ]; then
+		echo "Creating multi-arch manifest"
+		"$BUILD_CMD" manifest create "wubloader-$component"
+	fi
 	latest="$BASE/wubloader-$component:latest"
 	specific="$BASE/wubloader-$component:$TAG"
+	# no way to build multi-arch in parallel, so just build amd64 first and then arm64.
+	# caching should work with both, so that'll make things less painful at least.
 	"$BUILD_CMD" build \
 		-f "$component/Dockerfile" \
 		-t "$latest" \
 		-t "$specific" \
+		--manifest "wubloader-$component"
+		--arch amd64
+		"${CACHE_ARGS[@]}" \
+		.
+	"$BUILD_CMD" build \
+		-f "$component/Dockerfile" \
+		-t "$latest" \
+		-t "$specific" \
+		--manifest "wubloader-$component"
+		--arch arm64
 		"${CACHE_ARGS[@]}" \
 		.
 	echo "Built image wubloader-$component:$TAG"
-	if [ -n "$PUSH" ]; then
-		echo "Pushing tag $specific"
-		"$BUILD_CMD" push "$specific"
-	fi
-	if [ "$PUSH" == "latest" ]; then
-		echo "Pushing tag $latest"
-		"$BUILD_CMD" push "$latest"
+	if [ -n "$MULTIPLATFORM ]; then
+		# manifest pushes are buildah-specific. docker buildx multiplatform builds have different syntax...
+		if [ -n "$PUSH" ]; then
+			echo "Pushing tag $specific"
+			"BUILD_CMD" manifest push --all "docker://$specific"
+		fi
+		if [ "$PUSH" == "latest" ]; then
+			echo "Pushing tag $latest"
+			"$BUILD_CMD" manifest push --all "docker://$latest"
+		fi
+	else
+		if [ -n "$PUSH" ]; then
+			echo "Pushing tag $specific"
+			"$BUILD_CMD" push "$specific"
+		fi
+		if [ "$PUSH" == "latest" ]; then
+			echo "Pushing tag $latest"
+			"$BUILD_CMD" push "$latest"
+		fi
 	fi
 done

From 2e877165927a54b065e99f1eca885f4280812714 Mon Sep 17 00:00:00 2001
From: Hubbe <HubbeKing@users.noreply.github.com>
Date: Fri, 8 Nov 2024 09:16:58 +0200
Subject: [PATCH 03/28] Fix typo

Missed a "
---
 build | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/build b/build
index aef3912..14e212d 100755
--- a/build
+++ b/build
@@ -98,7 +98,7 @@ for component in "${COMPONENTS[@]}"; do
 		"${CACHE_ARGS[@]}" \
 		.
 	echo "Built image wubloader-$component:$TAG"
-	if [ -n "$MULTIPLATFORM ]; then
+	if [ -n "$MULTIPLATFORM" ]; then
 		# manifest pushes are buildah-specific. docker buildx multiplatform builds have different syntax...
 		if [ -n "$PUSH" ]; then
 			echo "Pushing tag $specific"

From cf289e869541653823b3ce01857adbfba61e46b1 Mon Sep 17 00:00:00 2001
From: Hubbe <HubbeKing@users.noreply.github.com>
Date: Fri, 8 Nov 2024 09:23:04 +0200
Subject: [PATCH 04/28] Fix multi-line commands

Just completely forgot how to bash there...
---
 build | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/build b/build
index 14e212d..a194929 100755
--- a/build
+++ b/build
@@ -85,16 +85,16 @@ for component in "${COMPONENTS[@]}"; do
 		-f "$component/Dockerfile" \
 		-t "$latest" \
 		-t "$specific" \
-		--manifest "wubloader-$component"
-		--arch amd64
+		--manifest "wubloader-$component" \
+		--arch amd64 \
 		"${CACHE_ARGS[@]}" \
 		.
 	"$BUILD_CMD" build \
 		-f "$component/Dockerfile" \
 		-t "$latest" \
 		-t "$specific" \
-		--manifest "wubloader-$component"
-		--arch arm64
+		--manifest "wubloader-$component" \
+		--arch arm64 \
 		"${CACHE_ARGS[@]}" \
 		.
 	echo "Built image wubloader-$component:$TAG"

From 61a6ccff0f27a1663d9697aa4abf06e16d6ad49d Mon Sep 17 00:00:00 2001
From: Hubbe <HubbeKing@users.noreply.github.com>
Date: Fri, 8 Nov 2024 09:25:40 +0200
Subject: [PATCH 05/28] Fix missing $

I'm not good at computer
---
 build | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/build b/build
index a194929..dda22ee 100755
--- a/build
+++ b/build
@@ -102,7 +102,7 @@ for component in "${COMPONENTS[@]}"; do
 		# manifest pushes are buildah-specific. docker buildx multiplatform builds have different syntax...
 		if [ -n "$PUSH" ]; then
 			echo "Pushing tag $specific"
-			"BUILD_CMD" manifest push --all "docker://$specific"
+			"$BUILD_CMD" manifest push --all "docker://$specific"
 		fi
 		if [ "$PUSH" == "latest" ]; then
 			echo "Pushing tag $latest"

From 8a79d0977fda171b03f8099089df7020b0998594 Mon Sep 17 00:00:00 2001
From: Hubbe <HubbeKing@users.noreply.github.com>
Date: Fri, 8 Nov 2024 09:47:53 +0200
Subject: [PATCH 06/28] Skip building buscribe under arm64

Also handle not setting MULTIPLATFORM properly
---
 build | 59 ++++++++++++++++++++++++++++++++++-------------------------
 1 file changed, 34 insertions(+), 25 deletions(-)

diff --git a/build b/build
index dda22ee..ee7dc47 100755
--- a/build
+++ b/build
@@ -73,42 +73,51 @@ fi
 		
 for component in "${COMPONENTS[@]}"; do
 	echo "Building image for $component"
-	if [ -n "$MULTIPLATFORM" ]; then
-		echo "Creating multi-arch manifest"
-		"$BUILD_CMD" manifest create "wubloader-$component"
-	fi
 	latest="$BASE/wubloader-$component:latest"
 	specific="$BASE/wubloader-$component:$TAG"
-	# no way to build multi-arch in parallel, so just build amd64 first and then arm64.
-	# caching should work with both, so that'll make things less painful at least.
-	"$BUILD_CMD" build \
-		-f "$component/Dockerfile" \
-		-t "$latest" \
-		-t "$specific" \
-		--manifest "wubloader-$component" \
-		--arch amd64 \
-		"${CACHE_ARGS[@]}" \
-		.
-	"$BUILD_CMD" build \
-		-f "$component/Dockerfile" \
-		-t "$latest" \
-		-t "$specific" \
-		--manifest "wubloader-$component" \
-		--arch arm64 \
-		"${CACHE_ARGS[@]}" \
-		.
-	echo "Built image wubloader-$component:$TAG"
 	if [ -n "$MULTIPLATFORM" ]; then
+		echo "Creating multi-arch manifest"
+		"$BUILD_CMD" manifest create "wubloader-$component"
+		# no way to build multi-arch in parallel, so just build amd64 first and then arm64.
+		# caching should work with both, so that'll make things less painful at least.
+		"$BUILD_CMD" build \
+			-f "$component/Dockerfile" \
+			-t "$latest" \
+			-t "$specific" \
+			--manifest "wubloader-$component" \
+			--arch amd64 \
+			"${CACHE_ARGS[@]}" \
+			.
+		# buscribe doesn't (yet?) build under arm64 due to pulling in libc-bin, skip arm64 for it
+		if [ "$component" != "buscribe ]; then
+			"$BUILD_CMD" build \
+				-f "$component/Dockerfile" \
+				-t "$latest" \
+				-t "$specific" \
+				--manifest "wubloader-$component" \
+				--arch arm64 \
+				"${CACHE_ARGS[@]}" \
+				.
+		fi
+		echo "Built image wubloader-$component:$TAG"
 		# manifest pushes are buildah-specific. docker buildx multiplatform builds have different syntax...
 		if [ -n "$PUSH" ]; then
 			echo "Pushing tag $specific"
-			"$BUILD_CMD" manifest push --all "docker://$specific"
+			"$BUILD_CMD" manifest push --all "$specific"
 		fi
 		if [ "$PUSH" == "latest" ]; then
 			echo "Pushing tag $latest"
-			"$BUILD_CMD" manifest push --all "docker://$latest"
+			"$BUILD_CMD" manifest push --all "$latest"
 		fi
 	else
+		"$BUILD_CMD" build \
+			-f "$component/Dockerfile" \
+			-t "$latest" \
+			-t "$specific" \
+			--manifest "wubloader-$component" \
+			"${CACHE_ARGS[@]}" \
+			.
+		echo "Built image wubloader-$component:$TAG"
 		if [ -n "$PUSH" ]; then
 			echo "Pushing tag $specific"
 			"$BUILD_CMD" push "$specific"

From 4749520db34bb2f42699b93141864ae0fb0d7d37 Mon Sep 17 00:00:00 2001
From: Hubbe <HubbeKing@users.noreply.github.com>
Date: Fri, 8 Nov 2024 09:50:28 +0200
Subject: [PATCH 07/28] Fix typo

I KEEP MISSING "
---
 build | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/build b/build
index ee7dc47..1f209d4 100755
--- a/build
+++ b/build
@@ -89,7 +89,7 @@ for component in "${COMPONENTS[@]}"; do
 			"${CACHE_ARGS[@]}" \
 			.
 		# buscribe doesn't (yet?) build under arm64 due to pulling in libc-bin, skip arm64 for it
-		if [ "$component" != "buscribe ]; then
+		if [ "$component" != "buscribe" ]; then
 			"$BUILD_CMD" build \
 				-f "$component/Dockerfile" \
 				-t "$latest" \

From 7e73e594bd167ca05d3b3c77a07d3009f95b9ab8 Mon Sep 17 00:00:00 2001
From: Hubbe <HubbeKing@users.noreply.github.com>
Date: Fri, 8 Nov 2024 10:03:20 +0200
Subject: [PATCH 08/28] Use --manifest, --platform, and --jobs to parallelize

---
 build | 62 +++++++++++++++++++++++++----------------------------------
 1 file changed, 26 insertions(+), 36 deletions(-)

diff --git a/build b/build
index 1f209d4..03650ef 100755
--- a/build
+++ b/build
@@ -76,38 +76,28 @@ for component in "${COMPONENTS[@]}"; do
 	latest="$BASE/wubloader-$component:latest"
 	specific="$BASE/wubloader-$component:$TAG"
 	if [ -n "$MULTIPLATFORM" ]; then
-		echo "Creating multi-arch manifest"
-		"$BUILD_CMD" manifest create "wubloader-$component"
-		# no way to build multi-arch in parallel, so just build amd64 first and then arm64.
-		# caching should work with both, so that'll make things less painful at least.
-		"$BUILD_CMD" build \
-			-f "$component/Dockerfile" \
-			-t "$latest" \
-			-t "$specific" \
-			--manifest "wubloader-$component" \
-			--arch amd64 \
-			"${CACHE_ARGS[@]}" \
-			.
+		echo "Creating multi-arch manifests"
+		"$BUILD_CMD" manifest create "$specific"
+		"$BUILD_CMD" manifest create "$latest"
+
 		# buscribe doesn't (yet?) build under arm64 due to pulling in libc-bin, skip arm64 for it
 		if [ "$component" != "buscribe" ]; then
 			"$BUILD_CMD" build \
-				-f "$component/Dockerfile" \
-				-t "$latest" \
-				-t "$specific" \
-				--manifest "wubloader-$component" \
-				--arch arm64 \
+				-f "$component/Dockerfile' \
+				--manifest "$specific" \
+				--manifest "$latest" \
+				--jobs 2 \
+				--platform=linux/amd64,linux/arm64 \
+				"${CACHE_ARGS[@]}" \
+				.
+		else
+			"$BUILD_CMD" build \
+				-f "$component/Dockerfile' \
+				--manifest "$specific" \
+				--manifest "$latest" \
+				--platform=linux/amd64 \
 				"${CACHE_ARGS[@]}" \
 				.
-		fi
-		echo "Built image wubloader-$component:$TAG"
-		# manifest pushes are buildah-specific. docker buildx multiplatform builds have different syntax...
-		if [ -n "$PUSH" ]; then
-			echo "Pushing tag $specific"
-			"$BUILD_CMD" manifest push --all "$specific"
-		fi
-		if [ "$PUSH" == "latest" ]; then
-			echo "Pushing tag $latest"
-			"$BUILD_CMD" manifest push --all "$latest"
 		fi
 	else
 		"$BUILD_CMD" build \
@@ -117,14 +107,14 @@ for component in "${COMPONENTS[@]}"; do
 			--manifest "wubloader-$component" \
 			"${CACHE_ARGS[@]}" \
 			.
-		echo "Built image wubloader-$component:$TAG"
-		if [ -n "$PUSH" ]; then
-			echo "Pushing tag $specific"
-			"$BUILD_CMD" push "$specific"
-		fi
-		if [ "$PUSH" == "latest" ]; then
-			echo "Pushing tag $latest"
-			"$BUILD_CMD" push "$latest"
-		fi
+	fi
+	echo "Built image wubloader-$component:$TAG"
+	if [ -n "$PUSH" ]; then
+		echo "Pushing tag $specific"
+		"$BUILD_CMD" push "$specific"
+	fi
+	if [ "$PUSH" == "latest" ]; then
+		echo "Pushing tag $latest"
+		"$BUILD_CMD" push "$latest"
 	fi
 done

From 3ffab22f1cf3dc4478f8052cce9b50d6fac1c19d Mon Sep 17 00:00:00 2001
From: Hubbe <HubbeKing@users.noreply.github.com>
Date: Fri, 8 Nov 2024 10:08:08 +0200
Subject: [PATCH 09/28] TYPOS AGAIN

why are ' and " the same button when shift keys are so futzy
---
 build | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/build b/build
index 03650ef..581908e 100755
--- a/build
+++ b/build
@@ -83,7 +83,7 @@ for component in "${COMPONENTS[@]}"; do
 		# buscribe doesn't (yet?) build under arm64 due to pulling in libc-bin, skip arm64 for it
 		if [ "$component" != "buscribe" ]; then
 			"$BUILD_CMD" build \
-				-f "$component/Dockerfile' \
+				-f "$component/Dockerfile" \
 				--manifest "$specific" \
 				--manifest "$latest" \
 				--jobs 2 \
@@ -92,7 +92,7 @@ for component in "${COMPONENTS[@]}"; do
 				.
 		else
 			"$BUILD_CMD" build \
-				-f "$component/Dockerfile' \
+				-f "$component/Dockerfile" \
 				--manifest "$specific" \
 				--manifest "$latest" \
 				--platform=linux/amd64 \

From a1337b2d0ab578c31f8db97fff999aa299e6aece Mon Sep 17 00:00:00 2001
From: Hubbe <HubbeKing@users.noreply.github.com>
Date: Fri, 8 Nov 2024 12:22:48 +0200
Subject: [PATCH 10/28] Try updating buildah package

---
 .github/workflows/main.yml | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index b6c83c7..b5e72f4 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -57,6 +57,11 @@ jobs:
           sudo apt update
           sudo apt install -y qemu-user-static
 
+      - name: Update buildah to latest
+        run: |
+          sudo apt update
+          sudo apt install -y buildah
+
       - name: Log into ghcr.io
         uses: docker/login-action@v3
         with:

From ce3f91444e140e43a999f0601a439748205ec648 Mon Sep 17 00:00:00 2001
From: Hubbe <HubbeKing@users.noreply.github.com>
Date: Fri, 8 Nov 2024 12:48:47 +0200
Subject: [PATCH 11/28] Try using the buildah/stable image from redhat

---
 .github/workflows/main.yml | 12 +++++-------
 1 file changed, 5 insertions(+), 7 deletions(-)

diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index b5e72f4..2ce7496 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -20,6 +20,9 @@ permissions:
 jobs:
   build:
     runs-on: ubuntu-24.04
+    # run jobs in buildah containers, since ubuntu is bad at keeping packages updated
+    container:
+      image: quay.io/buildah/stable:latest
     strategy:
       fail-fast: false
       matrix:
@@ -53,14 +56,9 @@ jobs:
 
       - name: Install QEMU
         # qemu-user-static is used by buildah to do multiplatform builds
+        # the buildah image is fedora-based, hence we use dnf
         run: |
-          sudo apt update
-          sudo apt install -y qemu-user-static
-
-      - name: Update buildah to latest
-        run: |
-          sudo apt update
-          sudo apt install -y buildah
+          sudo dnf install -y qemu-user-static
 
       - name: Log into ghcr.io
         uses: docker/login-action@v3

From 418af418108be0dcae2326aa3f1c09b9d7157b34 Mon Sep 17 00:00:00 2001
From: Hubbe <HubbeKing@users.noreply.github.com>
Date: Fri, 8 Nov 2024 12:55:57 +0200
Subject: [PATCH 12/28] Install git in buildah container

---
 .github/workflows/main.yml | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index 2ce7496..b7a0ddf 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -43,6 +43,13 @@ jobs:
           - thrimshim
           - zulip_bots
     steps:
+      - name: Install git
+        # the buildah/stable image doesn't include git
+        # actions/checkout works without git, but not if you do submodules
+        # thus, we install git
+        run: |
+          sudo dnf install -y git
+
       - name: Check out repo
         uses: actions/checkout@v3
         with:

From 0eed0ad2d0d90cf244c96257336a7789fc7f0a3f Mon Sep 17 00:00:00 2001
From: Hubbe <HubbeKing@users.noreply.github.com>
Date: Fri, 8 Nov 2024 13:00:02 +0200
Subject: [PATCH 13/28] Use redhat-actions/podman-login@v1

We don't have docker in the buildah/stable image
---
 .github/workflows/main.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index b7a0ddf..8d3ddd0 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -68,7 +68,7 @@ jobs:
           sudo dnf install -y qemu-user-static
 
       - name: Log into ghcr.io
-        uses: docker/login-action@v3
+        uses: redhat-actions/podman-login@v1
         with:
           registry: ghcr.io
           username: ${{ github.actor }}

From 1e25d154045088636aecd0440c8a3574779c7a9d Mon Sep 17 00:00:00 2001
From: Hubbe <HubbeKing@users.noreply.github.com>
Date: Fri, 8 Nov 2024 13:03:32 +0200
Subject: [PATCH 14/28] Log in to ghcr.io manually using run.

---
 .github/workflows/main.yml | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index 8d3ddd0..6cd3882 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -68,11 +68,12 @@ jobs:
           sudo dnf install -y qemu-user-static
 
       - name: Log into ghcr.io
-        uses: redhat-actions/podman-login@v1
-        with:
+        env:
           registry: ghcr.io
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          buildah login "$registry" --username "$username" --password "$password"
 
       - name: Build ${{ matrix.component }}
         # always push

From b19fad22ce6e710482a85cb8b1b6ecd490889067 Mon Sep 17 00:00:00 2001
From: Hubbe <HubbeKing@users.noreply.github.com>
Date: Fri, 8 Nov 2024 13:13:49 +0200
Subject: [PATCH 15/28] Install podman and use podman-login action

---
 .github/workflows/main.yml | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index 6cd3882..832fd13 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -43,12 +43,13 @@ jobs:
           - thrimshim
           - zulip_bots
     steps:
-      - name: Install git
+      - name: Install git & podman
         # the buildah/stable image doesn't include git
         # actions/checkout works without git, but not if you do submodules
         # thus, we install git
+        # we need podman for the podman-login action unfortunately
         run: |
-          sudo dnf install -y git
+          sudo dnf install -y git podman
 
       - name: Check out repo
         uses: actions/checkout@v3
@@ -68,12 +69,11 @@ jobs:
           sudo dnf install -y qemu-user-static
 
       - name: Log into ghcr.io
-        env:
+        uses: redhat-actions/podman-login@v1
+        with:
           registry: ghcr.io
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
-        run: |
-          buildah login "$registry" --username "$username" --password "$password"
 
       - name: Build ${{ matrix.component }}
         # always push

From acf4169c79c169ca3957969e8e3fd2cc3b01becb Mon Sep 17 00:00:00 2001
From: Hubbe <HubbeKing@users.noreply.github.com>
Date: Fri, 8 Nov 2024 16:34:37 +0200
Subject: [PATCH 16/28] Attempt to manually build containers/auth.json file

---
 .github/workflows/main.yml | 15 +++++++++------
 1 file changed, 9 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index 832fd13..4335483 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -43,13 +43,12 @@ jobs:
           - thrimshim
           - zulip_bots
     steps:
-      - name: Install git & podman
+      - name: Install git
         # the buildah/stable image doesn't include git
         # actions/checkout works without git, but not if you do submodules
         # thus, we install git
-        # we need podman for the podman-login action unfortunately
         run: |
-          sudo dnf install -y git podman
+          sudo dnf install -y git
 
       - name: Check out repo
         uses: actions/checkout@v3
@@ -68,12 +67,16 @@ jobs:
         run: |
           sudo dnf install -y qemu-user-static
 
-      - name: Log into ghcr.io
-        uses: redhat-actions/podman-login@v1
-        with:
+      - name: Create containers/auth.json file
+        env:
           registry: ghcr.io
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          mkdir -p ~/.config/containers
+          echo "$username:$password" | base64 > tmp
+          echo "{"auths":{"ghcr.io":{"auth": $(cat tmp)}}}" > ~/.config/containers/auth.json
+          rm tmp
 
       - name: Build ${{ matrix.component }}
         # always push

From 9b079d1bac0fc29782e5a3eb6a5509ae4dca6e85 Mon Sep 17 00:00:00 2001
From: HubbeKing <github@hubbe.club>
Date: Fri, 8 Nov 2024 20:02:57 +0200
Subject: [PATCH 17/28] Go back to the intended use so we can debug with act

---
 .github/workflows/main.yml | 26 ++++++--------------------
 build                      |  1 -
 2 files changed, 6 insertions(+), 21 deletions(-)

diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index 4335483..a241179 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -20,9 +20,6 @@ permissions:
 jobs:
   build:
     runs-on: ubuntu-24.04
-    # run jobs in buildah containers, since ubuntu is bad at keeping packages updated
-    container:
-      image: quay.io/buildah/stable:latest
     strategy:
       fail-fast: false
       matrix:
@@ -43,15 +40,8 @@ jobs:
           - thrimshim
           - zulip_bots
     steps:
-      - name: Install git
-        # the buildah/stable image doesn't include git
-        # actions/checkout works without git, but not if you do submodules
-        # thus, we install git
-        run: |
-          sudo dnf install -y git
-
       - name: Check out repo
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           submodules: recursive
           # By default, for PR builds, Actions will check out a merge commit between the actual
@@ -63,20 +53,16 @@ jobs:
 
       - name: Install QEMU
         # qemu-user-static is used by buildah to do multiplatform builds
-        # the buildah image is fedora-based, hence we use dnf
         run: |
-          sudo dnf install -y qemu-user-static
+          sudo apt update
+          sudo apt install -y qemu-user-static
 
-      - name: Create containers/auth.json file
-        env:
+      - name: Log in to ghcr.io
+        uses: docker/login-action@v3
+        with:
           registry: ghcr.io
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
-        run: |
-          mkdir -p ~/.config/containers
-          echo "$username:$password" | base64 > tmp
-          echo "{"auths":{"ghcr.io":{"auth": $(cat tmp)}}}" > ~/.config/containers/auth.json
-          rm tmp
 
       - name: Build ${{ matrix.component }}
         # always push
diff --git a/build b/build
index 581908e..c17aa9c 100755
--- a/build
+++ b/build
@@ -104,7 +104,6 @@ for component in "${COMPONENTS[@]}"; do
 			-f "$component/Dockerfile" \
 			-t "$latest" \
 			-t "$specific" \
-			--manifest "wubloader-$component" \
 			"${CACHE_ARGS[@]}" \
 			.
 	fi

From b1ca0a0aad16591d553d8ed952b081eca2edc878 Mon Sep 17 00:00:00 2001
From: HubbeKing <github@hubbe.club>
Date: Fri, 8 Nov 2024 20:50:43 +0200
Subject: [PATCH 18/28] Try to explicitly push manifests, not images

---
 build | 33 ++++++++++++++++++++-------------
 1 file changed, 20 insertions(+), 13 deletions(-)

diff --git a/build b/build
index c17aa9c..9ed9e77 100755
--- a/build
+++ b/build
@@ -70,16 +70,12 @@ if [ -n "$MULTIPLATFORM" ]; then
 		exit 1
 	fi
 fi
-		
+
 for component in "${COMPONENTS[@]}"; do
 	echo "Building image for $component"
 	latest="$BASE/wubloader-$component:latest"
 	specific="$BASE/wubloader-$component:$TAG"
 	if [ -n "$MULTIPLATFORM" ]; then
-		echo "Creating multi-arch manifests"
-		"$BUILD_CMD" manifest create "$specific"
-		"$BUILD_CMD" manifest create "$latest"
-
 		# buscribe doesn't (yet?) build under arm64 due to pulling in libc-bin, skip arm64 for it
 		if [ "$component" != "buscribe" ]; then
 			"$BUILD_CMD" build \
@@ -108,12 +104,23 @@ for component in "${COMPONENTS[@]}"; do
 			.
 	fi
 	echo "Built image wubloader-$component:$TAG"
-	if [ -n "$PUSH" ]; then
-		echo "Pushing tag $specific"
-		"$BUILD_CMD" push "$specific"
-	fi
-	if [ "$PUSH" == "latest" ]; then
-		echo "Pushing tag $latest"
-		"$BUILD_CMD" push "$latest"
-	fi
+    if [ -n "$MULTIPLATFORM" ]; then
+	    if [ -n "$PUSH" ]; then
+		    echo "Pushing tag $specific"
+		    "$BUILD_CMD" manifest push "$specific"
+	    fi
+	    if [ "$PUSH" == "latest" ]; then
+		    echo "Pushing tag $latest"
+		    "$BUILD_CMD" manifest push "$latest"
+	    fi
+    else
+	    if [ -n "$PUSH" ]; then
+		    echo "Pushing tag $specific"
+		    "$BUILD_CMD" push "$specific"
+	    fi
+	    if [ "$PUSH" == "latest" ]; then
+		    echo "Pushing tag $latest"
+		    "$BUILD_CMD" push "$latest"
+	    fi
+    fi
 done

From 58d4d038b4ad9a84ee6f41080d5681f7df6fd6e0 Mon Sep 17 00:00:00 2001
From: HubbeKing <github@hubbe.club>
Date: Wed, 13 Nov 2024 09:09:23 +0200
Subject: [PATCH 19/28] Update buscribe to debian:12 to make it build on arm64

---
 build               | 29 +++++++++--------------------
 buscribe/Dockerfile |  4 +++-
 2 files changed, 12 insertions(+), 21 deletions(-)

diff --git a/build b/build
index 9ed9e77..33a6889 100755
--- a/build
+++ b/build
@@ -54,7 +54,7 @@ case "${CACHE:-}" in
 		BUILD_CMD="docker"
 		;;
 	readonly)
-		CACHE_ARGS+=("--cache-from" "$CACHE_NAME")
+		CACHE_ARGS+=("--layers" "--cache-from" "$CACHE_NAME")
 		BUILD_CMD="buildah"
 		;;
 	*)
@@ -76,25 +76,14 @@ for component in "${COMPONENTS[@]}"; do
 	latest="$BASE/wubloader-$component:latest"
 	specific="$BASE/wubloader-$component:$TAG"
 	if [ -n "$MULTIPLATFORM" ]; then
-		# buscribe doesn't (yet?) build under arm64 due to pulling in libc-bin, skip arm64 for it
-		if [ "$component" != "buscribe" ]; then
-			"$BUILD_CMD" build \
-				-f "$component/Dockerfile" \
-				--manifest "$specific" \
-				--manifest "$latest" \
-				--jobs 2 \
-				--platform=linux/amd64,linux/arm64 \
-				"${CACHE_ARGS[@]}" \
-				.
-		else
-			"$BUILD_CMD" build \
-				-f "$component/Dockerfile" \
-				--manifest "$specific" \
-				--manifest "$latest" \
-				--platform=linux/amd64 \
-				"${CACHE_ARGS[@]}" \
-				.
-		fi
+		"$BUILD_CMD" build \
+			-f "$component/Dockerfile" \
+			--manifest "$specific" \
+			--manifest "$latest" \
+			--jobs 2 \
+			--platform=linux/amd64,linux/arm64 \
+			"${CACHE_ARGS[@]}" \
+			.
 	else
 		"$BUILD_CMD" build \
 			-f "$component/Dockerfile" \
diff --git a/buscribe/Dockerfile b/buscribe/Dockerfile
index 24e8eba..45496ce 100644
--- a/buscribe/Dockerfile
+++ b/buscribe/Dockerfile
@@ -1,4 +1,4 @@
-FROM debian:11
+FROM debian:12
 
 RUN apt update &&\
     apt install -y python3 libpq-dev python3-pip curl unzip ffmpeg
@@ -14,6 +14,8 @@ RUN cd /tmp \
 	&& unzip vosk-model-spk-0.4.zip -d /usr/share/buscribe \
 	&& rm *.zip
 
+RUN rm /usr/lib/python3.11/EXTERNALLY-MANAGED
+
 COPY common /tmp/common
 RUN pip install /tmp/common && rm -r /tmp/common
 

From b636b087ce0def36082e5dcfccb9e4b93887b4ab Mon Sep 17 00:00:00 2001
From: HubbeKing <github@hubbe.club>
Date: Wed, 13 Nov 2024 09:20:59 +0200
Subject: [PATCH 20/28] Try to compile an up-to-date buildah as a separate
 job???

---
 .github/workflows/main.yml | 34 +++++++++++++++++++++++++++++++++-
 1 file changed, 33 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index a241179..0e5b0ee 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -16,8 +16,32 @@ permissions:
   contents: read
   packages: write
 
-# build components using a job matrix, so they can run in parallel
+
 jobs:
+  # compile buildah v1.38.0, so we can have a more up-to-date version than ubuntu 24.04 has in its repos
+  buildah:
+    runs-on: ubuntu-24.04
+    steps:
+      - name: Install buildah dependencies
+        run: |
+          sudo apt update
+          sudo apt install -y bats btrfs-progs git go-md2man golang libapparmor-dev libglib2.0-dev libgpgme11-dev libseccomp-dev libselinux1-dev make runc skopeo libbtrfs-dev
+      - name: Pull buildah sources
+        run: |
+          git clone https://github.com/containers/buildah
+          cd buildah
+          git checkout v1.38.0
+      - name: Build buildah
+        run: |
+          cd buildah
+          make
+          sudo make install
+      - name: Save buildah build artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: buildah-bin
+          path: bin/buildah
+  # build components using a job matrix, so they can run in parallel
   build:
     runs-on: ubuntu-24.04
     strategy:
@@ -40,6 +64,14 @@ jobs:
           - thrimshim
           - zulip_bots
     steps:
+      - name: Ensure local bin exists
+        run: |
+          mkdir -p ~/.local/bin
+      - name: Fetch buildah build artifact
+        uses: actions/download-artifact@v4
+        with:
+          name: buildah-bin
+          path: ~/.local/bin/buildah
       - name: Check out repo
         uses: actions/checkout@v4
         with:

From a4e97f9fcdab07115cd1ce3ac4219f76698982ff Mon Sep 17 00:00:00 2001
From: HubbeKing <github@hubbe.club>
Date: Wed, 13 Nov 2024 09:38:39 +0200
Subject: [PATCH 21/28] Make build job depend on buildah job

---
 .github/workflows/main.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index 0e5b0ee..beca2ff 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -43,6 +43,7 @@ jobs:
           path: bin/buildah
   # build components using a job matrix, so they can run in parallel
   build:
+    needs: buildah
     runs-on: ubuntu-24.04
     strategy:
       fail-fast: false

From 37b97dd970940dbc211aade795db4fee54fc6e0f Mon Sep 17 00:00:00 2001
From: HubbeKing <github@hubbe.club>
Date: Wed, 13 Nov 2024 09:43:28 +0200
Subject: [PATCH 22/28] Upload buildah binary from /usr/local/bin/buildah

---
 .github/workflows/main.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index beca2ff..f096eba 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -40,7 +40,7 @@ jobs:
         uses: actions/upload-artifact@v4
         with:
           name: buildah-bin
-          path: bin/buildah
+          path: /usr/local/bin/buildah
   # build components using a job matrix, so they can run in parallel
   build:
     needs: buildah

From 56509ff137e50a2f6de9b44abefd3c0decd00403 Mon Sep 17 00:00:00 2001
From: HubbeKing <github@hubbe.club>
Date: Wed, 13 Nov 2024 10:59:07 +0200
Subject: [PATCH 23/28] Check buildah version in build step to verify

---
 .github/workflows/main.yml | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index f096eba..45efb8d 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -65,14 +65,14 @@ jobs:
           - thrimshim
           - zulip_bots
     steps:
-      - name: Ensure local bin exists
-        run: |
-          mkdir -p ~/.local/bin
       - name: Fetch buildah build artifact
         uses: actions/download-artifact@v4
         with:
           name: buildah-bin
-          path: ~/.local/bin/buildah
+          path: /usr/local/bin/buildah
+      - name: Check buildah version
+        run: |
+          buildah version
       - name: Check out repo
         uses: actions/checkout@v4
         with:

From 0ffaed8262c784063ba83ba31052d325934bfdd4 Mon Sep 17 00:00:00 2001
From: HubbeKing <github@hubbe.club>
Date: Wed, 13 Nov 2024 11:13:56 +0200
Subject: [PATCH 24/28] Manually add /usr/local/bin to the PATH???

---
 .github/workflows/main.yml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index 45efb8d..eb36fcd 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -70,6 +70,9 @@ jobs:
         with:
           name: buildah-bin
           path: /usr/local/bin/buildah
+      - name: Ensure /usr/local/bin is on the PATH
+        run: |
+          echo "/usr/local/bin" >> "$GITHUB_PATH"
       - name: Check buildah version
         run: |
           buildah version

From 66b73e9ef4acc150805457c1e08ab9fb010d46ed Mon Sep 17 00:00:00 2001
From: HubbeKing <github@hubbe.club>
Date: Wed, 13 Nov 2024 19:19:55 +0200
Subject: [PATCH 25/28] Try to figure out what's up with the PATH

---
 .github/workflows/main.yml | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index eb36fcd..78fe026 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -72,9 +72,11 @@ jobs:
           path: /usr/local/bin/buildah
       - name: Ensure /usr/local/bin is on the PATH
         run: |
-          echo "/usr/local/bin" >> "$GITHUB_PATH"
+          export PATH="/usr/local/bin:$PATH"
       - name: Check buildah version
         run: |
+          echo "$PATH"
+          which buildah
           buildah version
       - name: Check out repo
         uses: actions/checkout@v4

From bd89add15af4d17d3dc38cd7c0fb76b01f7034b6 Mon Sep 17 00:00:00 2001
From: HubbeKing <github@hubbe.club>
Date: Wed, 20 Nov 2024 09:46:16 +0200
Subject: [PATCH 26/28] Attempt to overwrite packaged /usr/bin/buildah and set
 +x for it

---
 .github/workflows/main.yml | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index 78fe026..43db710 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -69,13 +69,13 @@ jobs:
         uses: actions/download-artifact@v4
         with:
           name: buildah-bin
-          path: /usr/local/bin/buildah
-      - name: Ensure /usr/local/bin is on the PATH
+          path: /usr/bin/buildah
+      - name: Ensure buildah binary has +x permissions set
         run: |
-          export PATH="/usr/local/bin:$PATH"
+          chmod +x /usr/bin/buildah
       - name: Check buildah version
         run: |
-          echo "$PATH"
+          ls -l /usr/bin/buildah
           which buildah
           buildah version
       - name: Check out repo

From 7184e5cadd0b7ae2d96b6abc7de1faecbd888134 Mon Sep 17 00:00:00 2001
From: HubbeKing <github@hubbe.club>
Date: Wed, 20 Nov 2024 09:54:07 +0200
Subject: [PATCH 27/28] Okay, try /usr/local/bin but set +x for it?

---
 .github/workflows/main.yml | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index 43db710..f9e926f 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -69,13 +69,14 @@ jobs:
         uses: actions/download-artifact@v4
         with:
           name: buildah-bin
-          path: /usr/bin/buildah
+          path: /usr/local/bin/buildah
       - name: Ensure buildah binary has +x permissions set
         run: |
-          chmod +x /usr/bin/buildah
+          chmod +x /usr/local/bin/buildah
       - name: Check buildah version
         run: |
-          ls -l /usr/bin/buildah
+          ls -l /usr/local/bin/buildah
+          echo $PATH
           which buildah
           buildah version
       - name: Check out repo

From 197d76de148e55de292ed21b3dd782e9e2667e81 Mon Sep 17 00:00:00 2001
From: Hubbe <HubbeKing@users.noreply.github.com>
Date: Wed, 4 Dec 2024 14:03:54 +0200
Subject: [PATCH 28/28] Sudo chmod??

Grasping at straws here...
---
 .github/workflows/main.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index f9e926f..6b4254a 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -72,7 +72,7 @@ jobs:
           path: /usr/local/bin/buildah
       - name: Ensure buildah binary has +x permissions set
         run: |
-          chmod +x /usr/local/bin/buildah
+          sudo chmod +x /usr/local/bin/buildah
       - name: Check buildah version
         run: |
           ls -l /usr/local/bin/buildah