feat(build): lower the Linux glibc floor to 2.31 and publish linux-arm64

.github/actions/setup-linux-build/action.yml

@@ -0,0 +1,85 @@
+# Prepares a bare old-glibc container to build the dig-node Linux artifacts, and DECLARES the
+# canonical glibc floor those artifacts must honour.
+#
+# WHY (#1736): a glibc-linked binary runs on its BUILD glibc and anything newer, never anything
+# older — so the builder image, and nothing else, sets the supported-OS floor. dig-node v0.64.0
+# silently began requiring glibc >= 2.38 (breaking Ubuntu 22.04 LTS, Debian 12 and Amazon Linux
+# 2023) purely because GitHub repointed `ubuntu-latest` at Ubuntu 24.04. Every job that produces a
+# shipped Linux artifact — the release binaries AND the apt `.deb` — therefore runs inside a PINNED
+# container and calls this action, so there is one floor rather than one per workflow.
+#
+# The job must declare `container: debian:11`; this action asserts that the container it is running
+# in really provides `glibc-floor`, so the image and the number cannot drift apart.
+#
+# Because this is a LOCAL composite action it can only run after `actions/checkout`, and checkout in
+# a bare container needs `git` + `curl` + CA certificates already present — so each calling job
+# bootstraps exactly those three packages first, then checks out, then calls this action.
+name: Set up an old-glibc Linux build
+description: Install the Rust + C toolchains in the pinned old-glibc container and assert its glibc matches the canonical floor.
+
+outputs:
+  glibc-floor:
+    description: "The canonical minimum glibc version every published dig-node Linux artifact must run on."
+    value: ${{ steps.floor.outputs.version }}
+
+runs:
+  using: composite
+  steps:
+    - name: Declare the canonical glibc floor
+      id: floor
+      shell: bash
+      # THE single source of truth for the floor. 2.31 (Debian 11) clears every distro #1736
+      # reported broken — Ubuntu 22.04 (2.35), Debian 12 (2.36), Amazon Linux 2023 (2.34) — plus
+      # Ubuntu 20.04 (2.31), Debian 11 (2.31) and RHEL 9 (2.34). Raising it is a DELIBERATE act:
+      # change this value, the `container:` image in every calling job, SPEC.md §11.3 and the
+      # published docs together — never one alone.
+      run: echo "version=2.31" >>"$GITHUB_OUTPUT"
+
+    - name: Install the Rust + C toolchains
+      shell: bash
+      # The container is a bare Debian 11 with neither toolchain. The C toolchain is not optional:
+      # `openssl-sys` builds vendored OpenSSL from source (needs perl + make) and several other
+      # -sys crates in the Chia-wallet-SDK graph want cmake/clang. `binutils` provides the
+      # `readelf` the floor gate reads.
+      run: |
+        set -eux
+        apt-get update -qq
+        apt-get install -y --no-install-recommends \
+          ca-certificates curl git build-essential pkg-config perl make cmake clang binutils
+
+    - name: Verify the container glibc IS the declared floor
+      shell: bash
+      run: |
+        set -eu
+        # NO PIPE. `ldd --version | head -n1` is a SIGPIPE race: `head` closes the pipe after the
+        # first line, `ldd` dies with SIGPIPE, and under `pipefail` the step fails with exit 141
+        # — intermittently, depending on whether ldd had already finished writing. It passed twice
+        # before failing once. Read the output into a variable and slice it with shell expansion.
+        ldd_output="$(ldd --version 2>&1)"
+        IFS= read -r first_line <<<"$ldd_output"
+        # glibc's banner ends with the bare version, e.g. "ldd (Debian GLIBC 2.31-13) 2.31".
+        actual="${first_line##* }"
+        declared="${{ steps.floor.outputs.version }}"
+        case "$actual" in
+          [0-9]*.[0-9]*) ;;
+          *)
+            echo "::error::could not parse a glibc version out of: $first_line"
+            exit 1
+            ;;
+        esac
+        echo "container glibc: $actual, declared floor: $declared"
+        test "$actual" = "$declared" || {
+          echo "::error::build container glibc $actual != the declared floor $declared. Update the floor in .github/actions/setup-linux-build/action.yml, every calling job's container: image, SPEC.md §11.3 and the docs TOGETHER."
+          exit 1
+        }
+
+    - name: Install Rust via rustup
+      shell: bash
+      # rustup, not Debian 11's rustc, which is far older than this workspace needs. Only the Rust
+      # toolchain comes from rustup — the glibc it links against is the container's, which is the
+      # entire point of the container.
+      run: |
+        set -eu
+        curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs \
+          | sh -s -- -y --default-toolchain stable --profile minimal --no-modify-path
+        echo "$HOME/.cargo/bin" >>"$GITHUB_PATH"

.github/actions/stage-binaries/action.yml

@@ -0,0 +1,53 @@
+# Stages the two release binaries under their CANONICAL asset names (SPEC §11.2) and uploads them.
+#
+# WHY this is a composite action rather than an inline step: the binaries are produced by TWO jobs
+# in build-binaries.yml — the Windows/macOS matrix on its host runners, and the Linux matrix inside
+# a pinned old-glibc container (#1736) — and the ONE thing that must never diverge between them is
+# the asset naming every downstream consumer resolves (the dig-installer thin-shim, apt.dig.net's
+# packaging template). Naming lives here, once.
+name: Stage release binaries
+description: Copy the built dig-node + dign binaries to dist/ under their canonical asset names and upload them.
+
+inputs:
+  version:
+    description: "Version string stamped into each artifact filename (e.g. `1.2.3` or `1.2.3-nightly.20260714.abc1234`)."
+    required: true
+  target:
+    description: "Rust target triple the binaries were built for (selects the target/<triple>/release dir)."
+    required: true
+  out-name:
+    description: "Canonical OS/arch asset suffix, e.g. `linux-x64`, `linux-arm64`, `windows-x64.exe`."
+    required: true
+  exe-suffix:
+    description: "Executable suffix of the built binary — `.exe` on Windows, empty elsewhere."
+    required: false
+    default: ""
+
+runs:
+  using: composite
+  steps:
+    - name: Stage artifact (dual-named)
+      shell: bash
+      run: |
+        set -euo pipefail
+        VER="${{ inputs.version }}"
+        SUFFIX="${{ inputs.exe-suffix }}"
+        mkdir -p dist
+        # `dig-node` is the service binary; `dign` is its FIRST-CLASS alias (issue #548) — a
+        # separate [[bin]] over the same entrypoint, published under its own stem so the
+        # installer can resolve it via Repo::dign(). Both are required; a missing one is a
+        # release-shaped failure, so fail loudly rather than publishing half the pair.
+        for stem in dig-node dign; do
+          src="target/${{ inputs.target }}/release/${stem}${SUFFIX}"
+          test -f "$src" || { echo "::error::binary not produced: $src"; exit 1; }
+          # out-name already carries the `.exe` on Windows, so it is never appended twice.
+          cp "$src" "dist/${stem}-${VER}-${{ inputs.out-name }}"
+        done
+        ls -la dist
+
+    - name: Upload build artifact
+      uses: actions/upload-artifact@v4
+      with:
+        name: dig-node-${{ inputs.out-name }}
+        path: dist/*
+        if-no-files-found: error

.github/workflows/build-binaries.yml

@@ -21,9 +21,21 @@
 # `dign` is a FIRST-CLASS alias binary (issue #548): a SEPARATE `[[bin]]` target sharing the SAME
 # entrypoint, published alongside `dig-node` under its own stem `dign-<ver>-<os>-<arch>[.exe]`.
 #
-# NO linux-arm64 asset is published: the Linux build graph pulls `openssl-sys` (via the Chia wallet
-# SDK), so an aarch64-Linux cross-compile would also have to cross-compile OpenSSL — fragile, and
-# no consumer requests it (SPEC §11.3).
+# THE LINUX TARGETS BUILD INSIDE A PINNED OLD-GLIBC CONTAINER (`debian:11`, glibc 2.31), in their
+# own `build-linux` job — NOT on the runner image (#1736). A glibc-linked binary runs on the build
+# glibc and anything NEWER, never anything older, so the builder image silently dictates the
+# supported-OS floor. When GitHub moved `ubuntu-latest` to Ubuntu 24.04 the v0.64.0 `linux-x64`
+# binary began requiring glibc >= 2.38 and stopped starting on Ubuntu 22.04 LTS (2.35), Debian 12
+# (2.36) and Amazon Linux 2023 (2.34) — the three most common server distros — with no code change
+# and no signal. Pinning the container makes the floor an explicit, reviewable decision, and
+# `scripts/check-glibc-floor.sh` FAILS the build if a produced binary exceeds it (#1741 covers the
+# arch half of the same problem).
+#
+# `linux-arm64` IS published (#1741): it builds NATIVELY on the free public-repo `ubuntu-24.04-arm`
+# runner, so the vendored `openssl-sys` in the Chia-wallet-SDK graph is compiled by a native
+# toolchain. The old SPEC §11.3 rationale for omitting arm64 was about CROSS-compiling OpenSSL;
+# a native arm64 runner removes that objection entirely. Graviton/Ampere/Pi hosts are exactly the
+# always-on machines a volunteer node runs on.
 #
 # The gate runs fmt + clippy ONLY — a PR's own CI (ci.yml) already ran the full
 # fmt/clippy/test/coverage set (`cargo llvm-cov nextest --workspace`) against the merged tree
@@ -83,6 +95,9 @@ jobs:
       - run: cargo fmt --all --check
       - run: cargo clippy --all-targets --locked -- -D warnings
 
+  # Windows + macOS build on their HOST runner images. Neither has a glibc floor to defend: the
+  # Windows binary targets the MSVC CRT and macOS pins its floor via the SDK/deployment target, so
+  # only the Linux targets need the pinned-container treatment (`build-linux` below).
   build:
     name: build (${{ matrix.out_name }})
     needs: check
@@ -92,23 +107,17 @@ jobs:
         include:
           - os: windows-latest
             target: x86_64-pc-windows-msvc
-            bin: dig-node.exe
+            exe_suffix: ".exe"
             out_name: windows-x64.exe
-          - os: ubuntu-latest
-            target: x86_64-unknown-linux-gnu
-            bin: dig-node
-            out_name: linux-x64
           # Both macOS arches build on the fast Apple-silicon macos-14 runner; the Intel binary is
           # CROSS-COMPILED there (host arm64 → target x86_64-apple-darwin). The only C/asm dep in
           # the macOS graph is `ring` (via rustls); `openssl-sys` is cfg-gated to non-Apple targets,
           # so there is no vendored-openssl cross-compile to worry about.
           - os: macos-14 # Apple silicon (arm64)
             target: aarch64-apple-darwin
-            bin: dig-node
             out_name: macos-arm64
           - os: macos-14 # Apple silicon — cross-compiles the Intel binary
             target: x86_64-apple-darwin
-            bin: dig-node
             out_name: macos-x64
     runs-on: ${{ matrix.os }}
     steps:
@@ -136,32 +145,147 @@ jobs:
         # Build BOTH bins — `dign` is the first-class alias (issue #548), published alongside
         # `dig-node` under its own asset stem below.
         run: cargo build --release --locked --target ${{ matrix.target }} --bin dig-node --bin dign
-      - name: Stage artifact (dual-named)
-        id: stage
-        # The caller's `version` is stamped into the filename verbatim: a plain `1.2.3` for a
-        # stable tag, or the synthesized `1.2.3-nightly.YYYYMMDD.<shortsha>` for a nightly.
-        shell: bash
+      # The caller's `version` is stamped into the filename verbatim: a plain `1.2.3` for a stable
+      # tag, or the synthesized `1.2.3-nightly.YYYYMMDD.<shortsha>` for a nightly. Naming lives in
+      # the composite action so this job and `build-linux` can never diverge on it.
+      - uses: ./.github/actions/stage-binaries
+        with:
+          version: ${{ inputs.version }}
+          target: ${{ matrix.target }}
+          out-name: ${{ matrix.out_name }}
+          exe-suffix: ${{ matrix.exe_suffix }}
+
+  # The LINUX targets, built NATIVELY inside a pinned old-glibc container so the supported-OS floor
+  # is a decision rather than a side effect of whatever image `ubuntu-latest` points at this month
+  # (#1736 — see the header). aarch64 builds on the free public-repo arm64 runner (#1741), which
+  # means the vendored `openssl-sys` in the Chia-wallet-SDK graph is never cross-compiled.
+  build-linux:
+    name: build (${{ matrix.out_name }})
+    needs: check
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - runner: ubuntu-latest
+            target: x86_64-unknown-linux-gnu
+            out_name: linux-x64
+          - runner: ubuntu-24.04-arm
+            target: aarch64-unknown-linux-gnu
+            out_name: linux-arm64
+    runs-on: ${{ matrix.runner }}
+    # Debian 11 = glibc 2.31 = the floor declared by `.github/actions/setup-linux-build`, which
+    # asserts the two agree — so bumping this image without bumping the floor fails the build
+    # instead of silently shipping a higher floor.
+    container: debian:11
+    steps:
+      # Bootstrap only what `actions/checkout` itself needs in a bare container; the full toolchain
+      # (and the floor assertion) comes from the local composite action, which cannot run until the
+      # repository is on disk.
+      - name: Bootstrap the container for checkout
+        run: |
+          set -eux
+          apt-get update -qq
+          apt-get install -y --no-install-recommends ca-certificates curl git
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.ref }}
+          persist-credentials: false
+      - name: Trust the checkout (git runs as root in the container)
+        run: git config --global --add safe.directory "$GITHUB_WORKSPACE"
+      - id: setup
+        uses: ./.github/actions/setup-linux-build
+      - run: rustc --version && cargo --version
+      - uses: actions/cache@v4
+        with:
+          path: |
+            ~/.cargo/registry/index
+            ~/.cargo/registry/cache
+            ~/.cargo/git/db
+            target
+          # The glibc floor is part of the cache key: artifacts built against a different glibc must
+          # never be restored into a build that claims a different floor.
+          key: linux-glibc${{ steps.setup.outputs.glibc-floor }}-build-${{ matrix.target }}-${{ hashFiles('**/Cargo.lock') }}
+          restore-keys: linux-glibc${{ steps.setup.outputs.glibc-floor }}-build-${{ matrix.target }}-
+      - name: Build release binaries (dig-node + dign alias)
+        # The triple is the HOST's here (native build, no cross-compilation), but it is passed
+        # explicitly anyway so the output lands in `target/<triple>/release/` — the one path the
+        # shared staging action reads on every platform.
+        run: cargo build --release --locked --target ${{ matrix.target }} --bin dig-node --bin dign
+      - uses: ./.github/actions/stage-binaries
+        with:
+          version: ${{ inputs.version }}
+          target: ${{ matrix.target }}
+          out-name: ${{ matrix.out_name }}
+      - name: Assert the glibc floor of the produced binaries
+        # The gate that makes the floor durable. Both binaries are checked: `dign` is a separate
+        # link, so checking only `dig-node` would leave the alias unguarded.
         run: |
-          VER="${{ inputs.version }}"
-          mkdir -p dist
-          SRC="target/${{ matrix.target }}/release/${{ matrix.bin }}"
-          test -f "$SRC" || { echo "binary not produced: $SRC"; exit 1; }
-          # On Windows out_name already ends in .exe; on unix there is no extension. Publish the
-          # binary under the canonical dig-node-* name every consumer resolves (#585 dropped the
-          # duplicate legacy dig-companion-* copy). See the header comment.
-          cp "$SRC" "dist/dig-node-${VER}-${{ matrix.out_name }}"
-          # The `dign` first-class alias (issue #548): a SEPARATE binary that behaves byte-for-byte
-          # like `dig-node`, published under its own `dign-*` stem (same shape as `dig-node-*`) so
-          # the dig-installer resolves it via Repo::dign(). Its filename differs from `dig-node`
-          # only by the `dig-node`->`dign` substring (incl. the `.exe` suffix on Windows).
-          DIGN_BIN="${{ matrix.bin }}"; DIGN_BIN="${DIGN_BIN/dig-node/dign}"
-          DIGN_SRC="target/${{ matrix.target }}/release/${DIGN_BIN}"
-          test -f "$DIGN_SRC" || { echo "dign binary not produced: $DIGN_SRC"; exit 1; }
-          cp "$DIGN_SRC" "dist/dign-${VER}-${{ matrix.out_name }}"
-          ls -la dist
-      - name: Upload build artifact
-        uses: actions/upload-artifact@v4
+          set -eu
+          for stem in dig-node dign; do
+            bin="dist/${stem}-${{ inputs.version }}-${{ matrix.out_name }}"
+            bash scripts/check-glibc-floor.sh "$bin" "${{ steps.setup.outputs.glibc-floor }}"
+            # Prove the gate is not a no-op against a REAL binary, not just against the unit
+            # tests' stub readelf: every ELF references at least the base glibc version
+            # (2.2.5 on x86_64, 2.17 on aarch64), so a floor of 2.0 MUST be rejected. A gate that
+            # cannot fail is a rubber stamp.
+            if bash scripts/check-glibc-floor.sh "$bin" 2.0 >/dev/null 2>&1; then
+              echo "::error::check-glibc-floor.sh accepted an impossible 2.0 floor for $bin — the gate is not working"
+              exit 1
+            fi
+          done
+
+  # PROOF, not inference: start each published Linux binary on the OLDEST distros it claims to
+  # support. `ldd --version` on the builder says what the binary was linked against; only executing
+  # it on the target says it STARTS there. #1736 was found by a binary that died at exec time.
+  #
+  # WHY `--version`: it is the one invocation that touches no config, no network, no filesystem
+  # state and no service manager, so it isolates "does the dynamic loader resolve this binary".
+  verify-linux-floor:
+    name: verify (${{ matrix.out_name }} on ${{ matrix.image }})
+    needs: build-linux
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          # The three distros #1736 reported broken, each on its own arch's runner so the binary
+          # executes natively rather than under emulation.
+          - runner: ubuntu-latest
+            out_name: linux-x64
+            image: ubuntu:22.04
+          - runner: ubuntu-latest
+            out_name: linux-x64
+            image: debian:12
+          - runner: ubuntu-latest
+            out_name: linux-x64
+            image: amazonlinux:2023
+          - runner: ubuntu-24.04-arm
+            out_name: linux-arm64
+            image: ubuntu:22.04
+          - runner: ubuntu-24.04-arm
+            out_name: linux-arm64
+            image: debian:12
+          - runner: ubuntu-24.04-arm
+            out_name: linux-arm64
+            image: amazonlinux:2023
+    runs-on: ${{ matrix.runner }}
+    steps:
+      - uses: actions/download-artifact@v4
         with:
           name: dig-node-${{ matrix.out_name }}
-          path: dist/*
-          if-no-files-found: error
+          path: dist
+      - name: Run both binaries on ${{ matrix.image }}
+        run: |
+          set -eu
+          chmod +x dist/*
+          for stem in dig-node dign; do
+            bin="${stem}-${{ inputs.version }}-${{ matrix.out_name }}"
+            echo "::group::$bin on ${{ matrix.image }}"
+            # `ldd` FIRST and unconditionally: the dynamic loader names only the FIRST
+            # unsatisfied library, so a partial fix reads as a brand-new regression. Logging the
+            # full dependency set makes the real remaining gap visible in one run.
+            docker run --rm -v "$PWD/dist:/dist:ro" "${{ matrix.image }}" \
+              sh -c "ldd /dist/$bin || true"
+            docker run --rm -v "$PWD/dist:/dist:ro" "${{ matrix.image }}" \
+              /dist/$bin --version
+            echo "::endgroup::"
+          done