From 47bdcb1ce05b159164aac09edd832f546d77a30f Mon Sep 17 00:00:00 2001
From: "austin.barrington" <austin@hyperbyte.dev>
Date: Fri, 12 Jun 2026 14:44:44 +0100
Subject: [PATCH 01/12] feat(release): optimise the release builds

---
 .github/scripts/run-in-rust-container.sh |   9 ++
 .github/workflows/release.yml            | 122 ++++++++++++++---------
 Dockerfile.runtime                       |  54 ++++++++++
 3 files changed, 138 insertions(+), 47 deletions(-)
 create mode 100644 Dockerfile.runtime

diff --git a/.github/scripts/run-in-rust-container.sh b/.github/scripts/run-in-rust-container.sh
index 0bed396..727a28c 100755
--- a/.github/scripts/run-in-rust-container.sh
+++ b/.github/scripts/run-in-rust-container.sh
@@ -33,6 +33,15 @@ docker_args=(
   -e "CARGO_INCREMENTAL=${CARGO_INCREMENTAL:-0}"
 )
 
+# Optional emulated platform (e.g. linux/arm64). Used by the release pipeline to
+# build aarch64 binaries on the amd64 runner via QEMU/binfmt. Requires
+# docker/setup-qemu-action to have registered binfmt handlers first. sccache keys
+# objects by target triple, so emulated arm64 builds share the same node-local
+# SCCACHE_DIR as the native amd64 builds without collisions.
+if [ -n "${RUST_PLATFORM:-}" ]; then
+  docker_args+=(--platform "${RUST_PLATFORM}")
+fi
+
 if [ -n "${CARGO_HOME:-}" ]; then
   docker_args+=(-e "CARGO_HOME=${CARGO_HOME}" -v "${CARGO_HOME}:${CARGO_HOME}")
 fi
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index c516356..42933fd 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -2,15 +2,24 @@ name: Release
 
 # Release pipeline. On every `v*` tag push we:
 #
-#   1. Build a multi-arch (linux/amd64, linux/arm64) Docker image and push it
-#      to GHCR under ghcr.io/hyperbyte-cloud/hyperbytedb. arm64 layers are
-#      built under QEMU emulation on the amd64 self-hosted runner.
-#   2. Re-use the same BuildKit cache to export the `artifacts` Dockerfile
-#      stage per platform and package:
+#   1. Compile hyperbytedb + hyperbytedb-cli for linux/amd64 (native) and
+#      linux/arm64 (emulated under QEMU) by running cargo inside rust:bookworm
+#      via run-in-rust-container.sh — exactly how CI compiles. This bind-mounts
+#      the node-local sccache hostPath (/opt/sccache), so the release shares the
+#      same warm compilation cache as the CI jobs and only recompiles crates
+#      whose inputs changed. The previous pipeline compiled inside the Dockerfile
+#      with a BuildKit cache mount, which is never exported across the ephemeral
+#      ARC runner pods — so every release recompiled both arches from scratch,
+#      arm64 fully emulated. That cold double-build is what took ~1 hour.
+#   2. Assemble a multi-arch (linux/amd64, linux/arm64) Docker image from those
+#      PREBUILT binaries (Dockerfile.runtime does no compilation) and push it to
+#      GHCR under ghcr.io/hyperbyte-cloud/hyperbytedb.
+#   3. Package the prebuilt binaries + matching libchdb.so straight from the
+#      staging tree (plain tar, no second buildx export):
 #        - hyperbytedb-<tag>-linux-x86_64.tar.gz   (amd64 / x64)
 #        - hyperbytedb-<tag>-linux-aarch64.tar.gz (arm64)
-#      Each tarball contains hyperbytedb, hyperbytedb-cli, and matching libchdb.so, plus a sha256.
-#   3. Publish a GitHub Release for the tag with the tarballs attached.
+#      Each tarball contains hyperbytedb, hyperbytedb-cli, and libchdb.so, plus a sha256.
+#   4. Publish a GitHub Release for the tag with the tarballs attached.
 #
 # Jobs run directly on the ARC runner pod (not inside a job container) so
 # QEMU/binfmt registration and Docker Buildx talk to the dind sidecar reliably.
@@ -32,6 +41,16 @@ concurrency:
   cancel-in-progress: false
 
 env:
+  CARGO_TERM_COLOR: always
+  # sccache requires incremental compilation to be disabled.
+  CARGO_INCREMENTAL: 0
+  # Match the CI build job exactly (same flags => same sccache cache keys), so
+  # the amd64 release compile reuses the objects the CI `build` job already
+  # populated in the node-local /opt/sccache.
+  RUSTFLAGS: "-D warnings"
+  RUSTC_WRAPPER: sccache
+  SCCACHE_DIR: /opt/sccache
+  SCCACHE_CACHE_SIZE: 20G
   CHDB_RUST_REPO: https://github.com/hyperbyte-cloud/chdb-rust.git
   CHDB_RUST_REF: feat_arrow_insert
   REGISTRY: ghcr.io
@@ -63,6 +82,38 @@ jobs:
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v3
 
+      # --- Compile through the warm node-local sccache (shared with CI) -------
+      # Binaries + the arch-matched libchdb.so are dropped into a per-arch
+      # staging tree that Dockerfile.runtime and the tarball steps consume:
+      #   release-staging/{amd64,arm64}/{hyperbytedb,hyperbytedb-cli,libchdb.so}
+
+      - name: Build linux/amd64 binaries
+        run: |
+          rm -rf release-staging/amd64
+          bash .github/scripts/run-in-rust-container.sh \
+            'bash .github/scripts/rust-container-setup.sh && \
+             sccache --zero-stats && \
+             cargo build --release --locked -p hyperbytedb --bin hyperbytedb -p hyperbytedb-cli --bin hyperbytedb-cli && \
+             sccache --show-stats && \
+             mkdir -p release-staging/amd64 && \
+             cp target/release/hyperbytedb target/release/hyperbytedb-cli /usr/local/lib/libchdb.so release-staging/amd64/'
+
+      - name: Build linux/arm64 binaries (emulated, warm sccache)
+        env:
+          # Run the rust container under QEMU. sccache keys objects by target
+          # triple, so arm64 objects coexist with amd64 in the same cache dir.
+          RUST_PLATFORM: linux/arm64
+        run: |
+          rm -rf release-staging/arm64
+          bash .github/scripts/run-in-rust-container.sh \
+            'export CARGO_TARGET_DIR=target/arm64 && \
+             bash .github/scripts/rust-container-setup.sh && \
+             sccache --zero-stats && \
+             cargo build --release --locked -p hyperbytedb --bin hyperbytedb -p hyperbytedb-cli --bin hyperbytedb-cli && \
+             sccache --show-stats && \
+             mkdir -p release-staging/arm64 && \
+             cp target/arm64/release/hyperbytedb target/arm64/release/hyperbytedb-cli /usr/local/lib/libchdb.so release-staging/arm64/'
+
       # GHCR package was originally published with a PAT (scripts/docker-upload.sh).
       # Until the package is linked to this repo for Actions access, set the
       # repository variable GHCR_AUTH=pat and configure GHCR_USERNAME / GHCR_PAT
@@ -94,14 +145,14 @@ jobs:
             type=sha,prefix=
             type=raw,value=latest,enable=${{ github.ref_type == 'tag' }}
 
-      - name: Build and push multi-arch image (linux/amd64 + linux/arm64)
+      - name: Build and push multi-arch image from prebuilt binaries
         uses: docker/build-push-action@v6
         with:
-          # Parent context holds hyperbytedb/ (this checkout) + chdb-rust/.
-          # Dockerfile lives at the repo root, not hyperbytedb/Dockerfile — that
-          # path is only valid when building from a monorepo parent locally.
-          context: ..
-          file: Dockerfile
+          # Context is the per-arch staging tree built above. Dockerfile.runtime
+          # picks the right subdir via $TARGETARCH and does no compilation, so
+          # the only work under QEMU is the trivial debian apt layer.
+          context: release-staging
+          file: Dockerfile.runtime
           platforms: linux/amd64,linux/arm64
           push: true
           # provenance=false keeps the image manifest as a simple
@@ -113,51 +164,28 @@ jobs:
           cache-from: type=gha
           cache-to: type=gha,mode=max
 
-      - name: Package linux/x86_64 (amd64) release tarball
+      - name: Package release tarballs
         env:
           TAG: ${{ github.ref_name }}
         run: |
           set -euo pipefail
           mkdir -p out
-          rm -rf staging-x86_64
-          docker buildx build \
-            --platform linux/amd64 \
-            --target artifacts \
-            --output type=local,dest=./staging-x86_64 \
-            --provenance=false \
-            --cache-from type=gha \
-            -f Dockerfile \
-            ..
-          NAME="hyperbytedb-${TAG}-linux-x86_64"
-          tar -C staging-x86_64 -czf "out/${NAME}.tar.gz" hyperbytedb hyperbytedb-cli libchdb.so
-          ( cd out && sha256sum "${NAME}.tar.gz" > "${NAME}.tar.gz.sha256" )
-          ls -lh "out/${NAME}.tar.gz" "out/${NAME}.tar.gz.sha256"
-
-      - name: Package linux/aarch64 (arm64) release tarball
-        env:
-          TAG: ${{ github.ref_name }}
-        run: |
-          set -euo pipefail
-          rm -rf staging-aarch64
-          docker buildx build \
-            --platform linux/arm64 \
-            --target artifacts \
-            --output type=local,dest=./staging-aarch64 \
-            --provenance=false \
-            --cache-from type=gha \
-            -f Dockerfile \
-            ..
-          NAME="hyperbytedb-${TAG}-linux-aarch64"
-          tar -C staging-aarch64 -czf "out/${NAME}.tar.gz" hyperbytedb hyperbytedb-cli libchdb.so
-          ( cd out && sha256sum "${NAME}.tar.gz" > "${NAME}.tar.gz.sha256" )
-          ls -lh "out/${NAME}.tar.gz" "out/${NAME}.tar.gz.sha256"
+          # staging arch dir -> released arch name
+          declare -A ARCHES=( [amd64]=x86_64 [arm64]=aarch64 )
+          for src in "${!ARCHES[@]}"; do
+            dst="${ARCHES[$src]}"
+            NAME="hyperbytedb-${TAG}-linux-${dst}"
+            tar -C "release-staging/${src}" -czf "out/${NAME}.tar.gz" \
+              hyperbytedb hyperbytedb-cli libchdb.so
+            ( cd out && sha256sum "${NAME}.tar.gz" > "${NAME}.tar.gz.sha256" )
+            ls -lh "out/${NAME}.tar.gz" "out/${NAME}.tar.gz.sha256"
+          done
 
       - name: Verify release artifacts
         env:
           TAG: ${{ github.ref_name }}
         run: |
           set -euo pipefail
-          mkdir -p out
           for arch in x86_64 aarch64; do
             tarball="out/hyperbytedb-${TAG}-linux-${arch}.tar.gz"
             test -s "${tarball}"
diff --git a/Dockerfile.runtime b/Dockerfile.runtime
new file mode 100644
index 0000000..1a6c4a4
--- /dev/null
+++ b/Dockerfile.runtime
@@ -0,0 +1,54 @@
+# syntax=docker/dockerfile:1
+#
+# Release runtime image assembled from PREBUILT binaries — it performs no
+# compilation. The release workflow (.github/workflows/release.yml) compiles
+# hyperbytedb / hyperbytedb-cli for each architecture through the warm
+# node-local sccache (the same cache CI uses), drops the results into a
+# per-arch staging tree, and points this Dockerfile at that tree as its build
+# context:
+#
+#   release-staging/
+#     amd64/{hyperbytedb, hyperbytedb-cli, libchdb.so}
+#     arm64/{hyperbytedb, hyperbytedb-cli, libchdb.so}
+#
+# A single multi-platform `docker buildx build` then selects the right per-arch
+# directory via the BuildKit-provided $TARGETARCH (amd64 / arm64), so the only
+# work done under QEMU emulation is the trivial apt layer below — never a Rust
+# compile. For building the image from source (local / kind) use ./Dockerfile.
+FROM debian:bookworm-slim
+
+# Set automatically by buildx per target platform: "amd64" or "arm64". Selects
+# which staging subdirectory to copy from.
+ARG TARGETARCH
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    ca-certificates libstdc++6 curl \
+    && rm -rf /var/lib/apt/lists/*
+
+COPY ${TARGETARCH}/libchdb.so /usr/local/lib/libchdb.so
+RUN ldconfig
+
+COPY ${TARGETARCH}/hyperbytedb /usr/local/bin/hyperbytedb
+COPY ${TARGETARCH}/hyperbytedb-cli /usr/local/bin/hyperbytedb-cli
+
+RUN mkdir -p /var/lib/hyperbytedb/wal /var/lib/hyperbytedb/meta \
+             /var/lib/hyperbytedb/chdb /var/lib/hyperbytedb/raft
+
+ENV HYPERBYTEDB__STORAGE__WAL_DIR=/var/lib/hyperbytedb/wal \
+    HYPERBYTEDB__STORAGE__META_DIR=/var/lib/hyperbytedb/meta \
+    HYPERBYTEDB__CHDB__SESSION_DATA_PATH=/var/lib/hyperbytedb/chdb \
+    HYPERBYTEDB__CLUSTER__RAFT_DIR=/var/lib/hyperbytedb/raft \
+    HYPERBYTEDB__LOGGING__LEVEL=info
+
+# Cap glibc's per-thread heap arenas. Without this, glibc creates up to
+# ~8×CPU arenas of 64 MiB each — we observed 22 of them resident in
+# production, contributing ~200–300 MiB of fragmented anonymous RSS
+# that the application never asked for. `2` is the conventional
+# server-side Rust+Tokio setting: enough to avoid the single-arena
+# contention pathology, small enough that fragmentation stays bounded.
+ENV MALLOC_ARENA_MAX=2
+
+EXPOSE 8086
+
+ENTRYPOINT ["hyperbytedb"]
+CMD ["serve"]

From a11368ea0dabe868a963bcb2289b83c1d1d06554 Mon Sep 17 00:00:00 2001
From: "austin.barrington" <austin@hyperbyte.dev>
Date: Fri, 12 Jun 2026 15:01:20 +0100
Subject: [PATCH 02/12] fix: tarball error

---
 .github/workflows/release.yml | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index 42933fd..1aabf54 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -190,9 +190,13 @@ jobs:
             tarball="out/hyperbytedb-${TAG}-linux-${arch}.tar.gz"
             test -s "${tarball}"
             test -s "out/hyperbytedb-${TAG}-linux-${arch}.tar.gz.sha256"
-            tar -tzf "${tarball}" | grep -qx hyperbytedb
-            tar -tzf "${tarball}" | grep -qx hyperbytedb-cli
-            tar -tzf "${tarball}" | grep -qx libchdb.so
+            # Capture the listing once and match with here-strings. Piping tar
+            # into `grep -q` makes grep close the pipe on first match, killing
+            # tar with SIGPIPE (141) which `pipefail` would surface as a failure.
+            listing="$(tar -tzf "${tarball}")"
+            for entry in hyperbytedb hyperbytedb-cli libchdb.so; do
+              grep -qx "${entry}" <<<"${listing}"
+            done
           done
           ls -lh out/
 

From 8045fe79f46c7e8299b2e96025603cfa51e4bd45 Mon Sep 17 00:00:00 2001
From: "austin.barrington" <austin@hyperbyte.dev>
Date: Fri, 12 Jun 2026 15:02:14 +0100
Subject: [PATCH 03/12] tmp envoke CI for release to check

---
 .github/workflows/release.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index 1aabf54..7e30a00 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -26,6 +26,7 @@ name: Release
 # Requires containerMode.type=dind on the runner scale set.
 
 on:
+  branch:
   push:
     tags: ["v*"]
 

From a9b141384ef6d33b8145da900bc7ee135a5a5428 Mon Sep 17 00:00:00 2001
From: "austin.barrington" <austin@hyperbyte.dev>
Date: Fri, 12 Jun 2026 15:15:32 +0100
Subject: [PATCH 04/12] remove build from ci testing as it's a dupe

---
 .github/workflows/ci.yml      | 26 --------------------------
 .github/workflows/release.yml |  3 ++-
 2 files changed, 2 insertions(+), 27 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index d7025bd..09c42a4 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -113,29 +113,3 @@ jobs:
              cargo test -p hyperbytedb-cli && \
              sccache --show-stats'
 
-  build:
-    name: Build
-    runs-on: hyperbytedb-operator-controller
-    needs: [clippy, test]
-    steps:
-      - uses: actions/checkout@v4
-
-      - name: Checkout chdb-rust
-        run: bash .github/scripts/checkout-chdb-rust.sh
-
-      - name: Install CI dependencies
-        uses: ./.github/actions/install-ci-deps
-        with:
-          profile: k8s-runner
-
-      - uses: Swatinem/rust-cache@v2
-
-      - name: Build release binaries
-        run: |
-          bash .github/scripts/run-in-rust-container.sh \
-            'bash .github/scripts/rust-container-setup.sh && \
-             sccache --zero-stats && \
-             cargo build --release -p hyperbytedb --bin hyperbytedb -p hyperbytedb-cli --bin hyperbytedb-cli && \
-             sccache --show-stats && \
-             test -x target/release/hyperbytedb && \
-             test -x target/release/hyperbytedb-cli'
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index 7e30a00..c01bbde 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -26,8 +26,9 @@ name: Release
 # Requires containerMode.type=dind on the runner scale set.
 
 on:
-  branch:
   push:
+    branches:
+      - main
     tags: ["v*"]
 
 permissions:

From 0d0be6a79ccbb885ac0a0a6f645a71b979051eb2 Mon Sep 17 00:00:00 2001
From: "austin.barrington" <austin@hyperbyte.dev>
Date: Fri, 12 Jun 2026 16:26:27 +0100
Subject: [PATCH 05/12] force use

---
 Dockerfile | 1 +
 1 file changed, 1 insertion(+)

diff --git a/Dockerfile b/Dockerfile
index a403902..1b71708 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -111,3 +111,4 @@ EXPOSE 8086
 
 ENTRYPOINT ["hyperbytedb"]
 CMD ["serve"]
+

From cda386b4151fa4f7290ceee710d881870dd7c725 Mon Sep 17 00:00:00 2001
From: "austin.barrington" <austin@hyperbyte.dev>
Date: Fri, 12 Jun 2026 16:27:06 +0100
Subject: [PATCH 06/12] feat: release optimisation

---
 .github/workflows/release.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index c01bbde..0a9a284 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -28,7 +28,7 @@ name: Release
 on:
   push:
     branches:
-      - main
+      - feat_release_optimisation
     tags: ["v*"]
 
 permissions:

From 3948f874850fd6ad7c2e529970c034a9e5eddac3 Mon Sep 17 00:00:00 2001
From: "austin.barrington" <austin@hyperbyte.dev>
Date: Fri, 12 Jun 2026 16:53:28 +0100
Subject: [PATCH 07/12] try splitting into multiple steps

---
 .github/scripts/run-in-rust-container.sh |   2 +-
 .github/workflows/release.yml            | 191 ++++++++++++++---------
 Cargo.toml                               |   1 +
 Dockerfile.runtime                       |   7 +-
 4 files changed, 124 insertions(+), 77 deletions(-)

diff --git a/.github/scripts/run-in-rust-container.sh b/.github/scripts/run-in-rust-container.sh
index 727a28c..719930b 100755
--- a/.github/scripts/run-in-rust-container.sh
+++ b/.github/scripts/run-in-rust-container.sh
@@ -18,9 +18,9 @@ WORKSPACE="${GITHUB_WORKSPACE:?GITHUB_WORKSPACE is required}"
 # Mount the parent directory so path = "../../chdb-rust" resolves inside the container.
 WORKSPACE_PARENT="$(dirname "${WORKSPACE}")"
 
+# ARC runner pods use dind; GitHub-hosted runners have Docker on the host.
 if ! command -v docker >/dev/null 2>&1; then
   echo "docker is required but was not found on PATH." >&2
-  echo "Configure the ARC runner scale set with containerMode.type=dind." >&2
   exit 1
 fi
 
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index 0a9a284..ada01ce 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -1,95 +1,65 @@
 name: Release
 
-# Release pipeline. On every `v*` tag push we:
+# Release pipeline on GitHub-hosted runners. On every `v*` tag push we:
 #
-#   1. Compile hyperbytedb + hyperbytedb-cli for linux/amd64 (native) and
-#      linux/arm64 (emulated under QEMU) by running cargo inside rust:bookworm
-#      via run-in-rust-container.sh — exactly how CI compiles. This bind-mounts
-#      the node-local sccache hostPath (/opt/sccache), so the release shares the
-#      same warm compilation cache as the CI jobs and only recompiles crates
-#      whose inputs changed. The previous pipeline compiled inside the Dockerfile
-#      with a BuildKit cache mount, which is never exported across the ephemeral
-#      ARC runner pods — so every release recompiled both arches from scratch,
-#      arm64 fully emulated. That cold double-build is what took ~1 hour.
-#   2. Assemble a multi-arch (linux/amd64, linux/arm64) Docker image from those
-#      PREBUILT binaries (Dockerfile.runtime does no compilation) and push it to
-#      GHCR under ghcr.io/hyperbyte-cloud/hyperbytedb.
-#   3. Package the prebuilt binaries + matching libchdb.so straight from the
-#      staging tree (plain tar, no second buildx export):
-#        - hyperbytedb-<tag>-linux-x86_64.tar.gz   (amd64 / x64)
-#        - hyperbytedb-<tag>-linux-aarch64.tar.gz (arm64)
-#      Each tarball contains hyperbytedb, hyperbytedb-cli, and libchdb.so, plus a sha256.
-#   4. Publish a GitHub Release for the tag with the tarballs attached.
+#   1. build-amd64 / build-arm64 (parallel) — compile hyperbytedb + hyperbytedb-cli
+#      inside rust:bookworm via run-in-rust-container.sh (same path as CI). Each
+#      job uploads its per-arch staging tree as a workflow artifact.
+#   2. publish — assemble the multi-arch Docker image from those prebuilt binaries
+#      (Dockerfile.runtime does no compilation), push to GHCR, package tarballs,
+#      and create the GitHub Release.
 #
-# Jobs run directly on the ARC runner pod (not inside a job container) so
-# QEMU/binfmt registration and Docker Buildx talk to the dind sidecar reliably.
-# Requires containerMode.type=dind on the runner scale set.
+# Compilation caches use GitHub Actions cache (sccache + Swatinem/rust-cache) instead
+# of the self-hosted runner hostPath used by CI.
 
 on:
   push:
-    branches:
-      - feat_release_optimisation
     tags: ["v*"]
+  workflow_dispatch:
 
 permissions:
   contents: write   # GitHub Release upload
   packages: write   # GHCR image push
 
 concurrency:
-  # Don't cancel an in-flight release if a new tag is pushed. Releases are
-  # not safe to interrupt — we'd leave half-pushed image manifests and a
-  # half-populated GitHub Release behind.
   group: release-${{ github.ref }}
   cancel-in-progress: false
 
 env:
   CARGO_TERM_COLOR: always
-  # sccache requires incremental compilation to be disabled.
   CARGO_INCREMENTAL: 0
-  # Match the CI build job exactly (same flags => same sccache cache keys), so
-  # the amd64 release compile reuses the objects the CI `build` job already
-  # populated in the node-local /opt/sccache.
   RUSTFLAGS: "-D warnings"
   RUSTC_WRAPPER: sccache
-  SCCACHE_DIR: /opt/sccache
-  SCCACHE_CACHE_SIZE: 20G
+  SCCACHE_DIR: ${{ github.workspace }}/.cache/sccache
+  SCCACHE_CACHE_SIZE: 10G
   CHDB_RUST_REPO: https://github.com/hyperbyte-cloud/chdb-rust.git
   CHDB_RUST_REF: feat_arrow_insert
   REGISTRY: ghcr.io
-  # Canonical package; docs and operator manifests reference
-  # ghcr.io/hyperbyte-cloud/hyperbytedb. Hardcoded because github.repository
-  # is wrong when this workflow runs from a fork or a renamed root repo.
   IMAGE_NAME: hyperbyte-cloud/hyperbytedb
 
 jobs:
-  release:
-    name: Build & publish release
-    runs-on: hyperbytedb-operator-controller
+  build-amd64:
+    name: Build linux/amd64
+    runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
 
       - name: Checkout chdb-rust
         run: bash .github/scripts/checkout-chdb-rust.sh
 
-      - name: Install CI dependencies
-        uses: ./.github/actions/install-ci-deps
+      - name: Restore sccache
+        uses: actions/cache@v4
         with:
-          profile: k8s-runner
+          path: ${{ env.SCCACHE_DIR }}
+          key: sccache-release-amd64-${{ hashFiles('**/Cargo.lock') }}
+          restore-keys: |
+            sccache-release-amd64-
 
-      - name: Set up QEMU (binfmt for linux/arm64 builds)
-        uses: docker/setup-qemu-action@v3
+      - uses: Swatinem/rust-cache@v2
         with:
-          platforms: arm64
-
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
-
-      # --- Compile through the warm node-local sccache (shared with CI) -------
-      # Binaries + the arch-matched libchdb.so are dropped into a per-arch
-      # staging tree that Dockerfile.runtime and the tarball steps consume:
-      #   release-staging/{amd64,arm64}/{hyperbytedb,hyperbytedb-cli,libchdb.so}
+          shared-key: release-amd64
 
-      - name: Build linux/amd64 binaries
+      - name: Compile release binaries
         run: |
           rm -rf release-staging/amd64
           bash .github/scripts/run-in-rust-container.sh \
@@ -100,10 +70,41 @@ jobs:
              mkdir -p release-staging/amd64 && \
              cp target/release/hyperbytedb target/release/hyperbytedb-cli /usr/local/lib/libchdb.so release-staging/amd64/'
 
-      - name: Build linux/arm64 binaries (emulated, warm sccache)
+      - name: Upload amd64 staging artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: release-staging-amd64
+          path: release-staging/amd64/
+          if-no-files-found: error
+
+  build-arm64:
+    name: Build linux/arm64
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Checkout chdb-rust
+        run: bash .github/scripts/checkout-chdb-rust.sh
+
+      - name: Set up QEMU (binfmt for linux/arm64)
+        uses: docker/setup-qemu-action@v3
+        with:
+          platforms: arm64
+
+      - name: Restore sccache
+        uses: actions/cache@v4
+        with:
+          path: ${{ env.SCCACHE_DIR }}
+          key: sccache-release-arm64-${{ hashFiles('**/Cargo.lock') }}
+          restore-keys: |
+            sccache-release-arm64-
+
+      - uses: Swatinem/rust-cache@v2
+        with:
+          shared-key: release-arm64
+
+      - name: Compile release binaries (emulated)
         env:
-          # Run the rust container under QEMU. sccache keys objects by target
-          # triple, so arm64 objects coexist with amd64 in the same cache dir.
           RUST_PLATFORM: linux/arm64
         run: |
           rm -rf release-staging/arm64
@@ -116,10 +117,49 @@ jobs:
              mkdir -p release-staging/arm64 && \
              cp target/arm64/release/hyperbytedb target/arm64/release/hyperbytedb-cli /usr/local/lib/libchdb.so release-staging/arm64/'
 
-      # GHCR package was originally published with a PAT (scripts/docker-upload.sh).
-      # Until the package is linked to this repo for Actions access, set the
-      # repository variable GHCR_AUTH=pat and configure GHCR_USERNAME / GHCR_PAT
-      # org secrets (PAT needs write:packages). Otherwise GITHUB_TOKEN is used.
+      - name: Upload arm64 staging artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: release-staging-arm64
+          path: release-staging/arm64/
+          if-no-files-found: error
+
+  publish-image:
+    name: Publish container image
+    runs-on: ubuntu-latest
+    needs: [build-amd64, build-arm64]
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Download prebuilt binaries
+        uses: actions/download-artifact@v4
+        with:
+          name: release-staging-amd64
+          path: release-staging/amd64
+
+      - uses: actions/download-artifact@v4
+        with:
+          name: release-staging-arm64
+          path: release-staging/arm64
+
+      - name: Verify staging layout
+        run: |
+          set -euo pipefail
+          for arch in amd64 arm64; do
+            test -x "release-staging/${arch}/hyperbytedb"
+            test -x "release-staging/${arch}/hyperbytedb-cli"
+            test -s "release-staging/${arch}/libchdb.so"
+          done
+          ls -lh release-staging/*/*
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+        with:
+          platforms: arm64
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
       - name: Log in to GitHub Container Registry (PAT)
         if: vars.GHCR_AUTH == 'pat'
         uses: docker/login-action@v3
@@ -147,32 +187,42 @@ jobs:
             type=sha,prefix=
             type=raw,value=latest,enable=${{ github.ref_type == 'tag' }}
 
-      - name: Build and push multi-arch image from prebuilt binaries
+      - name: Build and push multi-arch image
         uses: docker/build-push-action@v6
         with:
-          # Context is the per-arch staging tree built above. Dockerfile.runtime
-          # picks the right subdir via $TARGETARCH and does no compilation, so
-          # the only work under QEMU is the trivial debian apt layer.
           context: release-staging
           file: Dockerfile.runtime
           platforms: linux/amd64,linux/arm64
           push: true
-          # provenance=false keeps the image manifest as a simple
-          # multi-arch index (no attestation index wrapper), which is what
-          # `docker pull --platform=...` and downstream tooling expect.
           provenance: false
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}
           cache-from: type=gha
           cache-to: type=gha,mode=max
 
+  publish-release:
+    name: Publish GitHub Release
+    runs-on: ubuntu-latest
+    needs: [build-amd64, build-arm64, publish-image]
+    if: github.ref_type == 'tag'
+    steps:
+      - name: Download prebuilt binaries
+        uses: actions/download-artifact@v4
+        with:
+          name: release-staging-amd64
+          path: release-staging/amd64
+
+      - uses: actions/download-artifact@v4
+        with:
+          name: release-staging-arm64
+          path: release-staging/arm64
+
       - name: Package release tarballs
         env:
           TAG: ${{ github.ref_name }}
         run: |
           set -euo pipefail
           mkdir -p out
-          # staging arch dir -> released arch name
           declare -A ARCHES=( [amd64]=x86_64 [arm64]=aarch64 )
           for src in "${!ARCHES[@]}"; do
             dst="${ARCHES[$src]}"
@@ -192,9 +242,6 @@ jobs:
             tarball="out/hyperbytedb-${TAG}-linux-${arch}.tar.gz"
             test -s "${tarball}"
             test -s "out/hyperbytedb-${TAG}-linux-${arch}.tar.gz.sha256"
-            # Capture the listing once and match with here-strings. Piping tar
-            # into `grep -q` makes grep close the pipe on first match, killing
-            # tar with SIGPIPE (141) which `pipefail` would surface as a failure.
             listing="$(tar -tzf "${tarball}")"
             for entry in hyperbytedb hyperbytedb-cli libchdb.so; do
               grep -qx "${entry}" <<<"${listing}"
diff --git a/Cargo.toml b/Cargo.toml
index ded153a..170d02b 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -3,6 +3,7 @@ members = ["hyperbytedb", "hyperbytedb-proxy", "hyperbytedb-cli"]
 resolver = "2"
 
 [profile.release]
+debug = 0
 lto = "thin"
 codegen-units = 1
 strip = "symbols"
diff --git a/Dockerfile.runtime b/Dockerfile.runtime
index 1a6c4a4..269ab66 100644
--- a/Dockerfile.runtime
+++ b/Dockerfile.runtime
@@ -2,10 +2,9 @@
 #
 # Release runtime image assembled from PREBUILT binaries — it performs no
 # compilation. The release workflow (.github/workflows/release.yml) compiles
-# hyperbytedb / hyperbytedb-cli for each architecture through the warm
-# node-local sccache (the same cache CI uses), drops the results into a
-# per-arch staging tree, and points this Dockerfile at that tree as its build
-# context:
+# hyperbytedb / hyperbytedb-cli per architecture in parallel on GitHub-hosted
+# runners, uploads staging artifacts, and points this Dockerfile at the merged
+# tree as its build context:
 #
 #   release-staging/
 #     amd64/{hyperbytedb, hyperbytedb-cli, libchdb.so}

From bebe3b3f4c388d2be650bb5031c0488fe60af1a5 Mon Sep 17 00:00:00 2001
From: "austin.barrington" <austin@hyperbyte.dev>
Date: Fri, 12 Jun 2026 16:54:43 +0100
Subject: [PATCH 08/12] force exec of release workflow

---
 .github/workflows/release.yml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index ada01ce..51db6ca 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -14,6 +14,8 @@ name: Release
 
 on:
   push:
+    branches:
+      - feat_release_optimisation
     tags: ["v*"]
   workflow_dispatch:
 

From b97812468f90e2f4eb1d4004c017c13e0cf2e2e0 Mon Sep 17 00:00:00 2001
From: "austin.barrington" <austin@hyperbyte.dev>
Date: Fri, 12 Jun 2026 16:58:40 +0100
Subject: [PATCH 09/12] build-arm64 on native arm64 runner

---
 .github/workflows/release.yml | 22 +++++++---------------
 1 file changed, 7 insertions(+), 15 deletions(-)

diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index 51db6ca..246bf35 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -2,9 +2,9 @@ name: Release
 
 # Release pipeline on GitHub-hosted runners. On every `v*` tag push we:
 #
-#   1. build-amd64 / build-arm64 (parallel) — compile hyperbytedb + hyperbytedb-cli
-#      inside rust:bookworm via run-in-rust-container.sh (same path as CI). Each
-#      job uploads its per-arch staging tree as a workflow artifact.
+#   1. build-amd64 / build-arm64 (parallel) — compile on native GitHub-hosted runners
+#      (ubuntu-latest + ubuntu-24.04-arm). Each job uploads its per-arch staging
+#      tree as a workflow artifact. arm64 is not emulated under QEMU.
 #   2. publish — assemble the multi-arch Docker image from those prebuilt binaries
 #      (Dockerfile.runtime does no compilation), push to GHCR, package tarballs,
 #      and create the GitHub Release.
@@ -81,18 +81,13 @@ jobs:
 
   build-arm64:
     name: Build linux/arm64
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04-arm
     steps:
       - uses: actions/checkout@v4
 
       - name: Checkout chdb-rust
         run: bash .github/scripts/checkout-chdb-rust.sh
 
-      - name: Set up QEMU (binfmt for linux/arm64)
-        uses: docker/setup-qemu-action@v3
-        with:
-          platforms: arm64
-
       - name: Restore sccache
         uses: actions/cache@v4
         with:
@@ -105,19 +100,16 @@ jobs:
         with:
           shared-key: release-arm64
 
-      - name: Compile release binaries (emulated)
-        env:
-          RUST_PLATFORM: linux/arm64
+      - name: Compile release binaries
         run: |
           rm -rf release-staging/arm64
           bash .github/scripts/run-in-rust-container.sh \
-            'export CARGO_TARGET_DIR=target/arm64 && \
-             bash .github/scripts/rust-container-setup.sh && \
+            'bash .github/scripts/rust-container-setup.sh && \
              sccache --zero-stats && \
              cargo build --release --locked -p hyperbytedb --bin hyperbytedb -p hyperbytedb-cli --bin hyperbytedb-cli && \
              sccache --show-stats && \
              mkdir -p release-staging/arm64 && \
-             cp target/arm64/release/hyperbytedb target/arm64/release/hyperbytedb-cli /usr/local/lib/libchdb.so release-staging/arm64/'
+             cp target/release/hyperbytedb target/release/hyperbytedb-cli /usr/local/lib/libchdb.so release-staging/arm64/'
 
       - name: Upload arm64 staging artifact
         uses: actions/upload-artifact@v4

From b2927c5f1df29e34ceff2dc428d1dedc06aa203f Mon Sep 17 00:00:00 2001
From: "austin.barrington" <austin@hyperbyte.dev>
Date: Fri, 12 Jun 2026 17:32:41 +0100
Subject: [PATCH 10/12] Fix release cache post-steps after Docker builds

Docker writes bind-mounted target/ and sccache as root on GitHub-hosted
runners, which caused hashFiles and cache save steps to fail even when
compilation and artifact upload succeeded.

Co-authored-by: Cursor <cursoragent@cursor.com>
---
 .github/scripts/run-in-rust-container.sh | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)

diff --git a/.github/scripts/run-in-rust-container.sh b/.github/scripts/run-in-rust-container.sh
index 719930b..31f81c4 100755
--- a/.github/scripts/run-in-rust-container.sh
+++ b/.github/scripts/run-in-rust-container.sh
@@ -68,3 +68,20 @@ if [ -n "${RUNNER_TOOL_CACHE:-}" ]; then
 fi
 
 docker run "${docker_args[@]}" "$RUST_IMAGE" bash -ec "$command"
+
+# Docker runs as root by default. Files written into bind-mounted workspace paths
+# are owned by root on the host, which breaks post-job steps (rust-cache save,
+# actions/cache hashFiles, etc.) on GitHub-hosted runners.
+if command -v sudo >/dev/null 2>&1; then
+  chown_cmd=(sudo chown -R "$(id -u):$(id -g)")
+else
+  chown_cmd=(chown -R "$(id -u):$(id -g)")
+fi
+for path in target release-staging; do
+  if [ -e "${WORKSPACE}/${path}" ]; then
+    "${chown_cmd[@]}" "${WORKSPACE}/${path}"
+  fi
+done
+if [ -n "${SCCACHE_DIR:-}" ] && [ -d "${SCCACHE_DIR}" ]; then
+  "${chown_cmd[@]}" "${SCCACHE_DIR}"
+fi

From ab7144172d8de2e44fd4a59b14f4c465e1419c7f Mon Sep 17 00:00:00 2001
From: "austin.barrington" <austin@hyperbyte.dev>
Date: Fri, 12 Jun 2026 17:48:42 +0100
Subject: [PATCH 11/12] Fix release staging checks after artifact download

GitHub artifact upload strips executable bits, so verify staging layout
failed on test -x even though binaries were present. Restore +x before
verify and tarball packaging.

Co-authored-by: Cursor <cursoragent@cursor.com>
---
 .github/workflows/release.yml | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index 246bf35..92a062d 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -140,8 +140,9 @@ jobs:
         run: |
           set -euo pipefail
           for arch in amd64 arm64; do
-            test -x "release-staging/${arch}/hyperbytedb"
-            test -x "release-staging/${arch}/hyperbytedb-cli"
+            chmod +x "release-staging/${arch}/hyperbytedb" "release-staging/${arch}/hyperbytedb-cli"
+            test -f "release-staging/${arch}/hyperbytedb"
+            test -f "release-staging/${arch}/hyperbytedb-cli"
             test -s "release-staging/${arch}/libchdb.so"
           done
           ls -lh release-staging/*/*
@@ -216,6 +217,9 @@ jobs:
           TAG: ${{ github.ref_name }}
         run: |
           set -euo pipefail
+          for arch in amd64 arm64; do
+            chmod +x "release-staging/${arch}/hyperbytedb" "release-staging/${arch}/hyperbytedb-cli"
+          done
           mkdir -p out
           declare -A ARCHES=( [amd64]=x86_64 [arm64]=aarch64 )
           for src in "${!ARCHES[@]}"; do

From d2c4dc89b4b262fa0cc39ca44afe39942dc5966b Mon Sep 17 00:00:00 2001
From: "austin.barrington" <austin@hyperbyte.dev>
Date: Fri, 12 Jun 2026 19:32:06 +0100
Subject: [PATCH 12/12] remove feat_release_optimisation branch from the
 release workflow

---
 .github/workflows/release.yml | 2 --
 1 file changed, 2 deletions(-)

diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index 92a062d..94a5b0a 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -14,8 +14,6 @@ name: Release
 
 on:
   push:
-    branches:
-      - feat_release_optimisation
     tags: ["v*"]
   workflow_dispatch: