Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

Fix #3503: the Linux cuda12 engine package carries sm_60/61/70; the source Dockerfile builds it

Niko1221366c8dbaaef6
diff --git a/Dockerfile b/Dockerfile
index 33cf7679d..f6560ee17 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -5,6 +5,8 @@
#
# docker build -t strata:src . # RTX 20/30/40/50 (compiles for each, ~25 min)
# docker build -t strata:src --build-arg CUDA_ARCHITECTURES=86 . # one card only (faster); 86 = RTX 30, 89 = RTX 40, 120 = RTX 50
+# docker build -t strata:src-cuda12 --build-arg CUDA_IMAGE=nvidia/cuda:12.9.1-devel-ubuntu24.04 \
+# --build-arg CUDA_ARCHITECTURES="60;61;70;75;80;86;89" --build-arg BUILD_VISION=0 . # GTX 10 / P100 / V100 (driver 575+)
# docker run --rm --gpus all -p 127.0.0.1:8080:8080 --ulimit memlock=-1 --shm-size 2g --stop-timeout 120 \
# -v strata-data:/data -e MODEL=IQ3_XXS -e API_KEY=change-me-please-123 strata:src
#
@@ -35,7 +37,9 @@ RUN /venv/bin/python -c "import setup; print(setup.get_llama_cpp())"
RUN set -e; \
real=""; for a in $(echo "$CUDA_ARCHITECTURES" | tr ';,' ' '); do real="$real;${a}-real"; done; real=${real#;}; \
last=$(echo "$CUDA_ARCHITECTURES" | tr ';,' ' ' | awk '{print $NF}'); \
+ sm60=OFF; for a in $(echo "$CUDA_ARCHITECTURES" | tr ';,' ' '); do if [ "$a" -lt 75 ]; then sm60=ON; fi; done; \
/venv/bin/cmake -G Ninja -S . -B build -DCMAKE_BUILD_TYPE=Release -DSTRATA_ENABLE_CUDA=ON -DSTRATA_BUILD_TESTS=OFF \
+ -DSTRATA_EXPERIMENTAL_SM60=$sm60 \
-DSTRATA_NATIVE_EXPERTS=ON -DSTRATA_PORTABLE=ON -DSTRATA_Q6K_EXPERTS=ON -DSTRATA_MMQ_KQUANTS=ON \
"-DCMAKE_CUDA_ARCHITECTURES=${real};${last}-virtual" -DSTRATA_GGML_DIR=/src/third_party/llama.cpp; \
/venv/bin/cmake --build build --target strata strata_kq -j "$(nproc)"; \
@@ -50,19 +54,21 @@ RUN set -e; \
[ -f build/libstrata_kq.so ] && cp build/libstrata_kq.so /stage/ || true; \
[ -f build-vision/bin/strata-vision ] && cp build-vision/bin/strata-vision /stage/ || true; \
ver="${ENGINE_VERSION:-$(sed -n 's/^[[:space:]]*project([^)]*VERSION[[:space:]]*\([0-9][0-9.]*\).*/\1/p' CMakeLists.txt | head -1)}"; \
- echo "{\"version\": \"$ver\", \"source\": \"docker build\", \"archs\": \"$CUDA_ARCHITECTURES\", \"cuda\": \"13.0\", \"portable\": true}" > /stage/BUILD.json; \
+ cudafull=$(nvcc --version | sed -n 's/.*release \([0-9][0-9]*\.[0-9][0-9]*\).*/\1/p' | head -1); cudamajor=${cudafull%%.*}; \
+ if [ "$cudamajor" = 13 ]; then mindriver=580.65; else mindriver=575.51; fi; \
+ echo "{\"version\": \"$ver\", \"source\": \"docker build\", \"archs\": \"$CUDA_ARCHITECTURES\", \"cuda\": \"$cudafull\", \"portable\": true}" > /stage/BUILD.json; \
python3 packaging/collect_libs.py --binary build/strata --search /usr/local/cuda/lib64 --out /stage --also 'libcublasLt.so*'; \
archs=""; for a in $(echo "$CUDA_ARCHITECTURES" | tr ';,' ' '); do archs="$archs --arch sm_$a"; done; \
last=$(echo "$CUDA_ARCHITECTURES" | tr ';,' ' ' | awk '{print $NF}'); \
python3 packaging/package_engine.py --src /stage --backend cuda --version "$ver" --out /engine $archs --arch "compute_$last" \
- --min-driver nvidia=580.65 --variant cuda13 --platform linux-x86_64 --build-id "docker-source-${ver}-cuda13.0" --include SYSTEM-LIBS.txt --include 'libstrata_kq*' \
+ --min-driver nvidia=$mindriver --variant cuda$cudamajor --platform linux-x86_64 --build-id "docker-source-${ver}-cuda${cudafull}" --include SYSTEM-LIBS.txt --include 'libstrata_kq*' \
--include strata-vision; \
for d in /engine/*/; do python3 packaging/package_engine.py --verify "$d"; python3 packaging/binary_deps.py "$d"; done
# ---- 2. the Terminal (the release archive, built here)
FROM ${RUNTIME_BASE} AS terminal
ENV DEBIAN_FRONTEND=noninteractive
-RUN apt-get update && apt-get install -y --no-install-recommends python3 ca-certificates && rm -rf /var/lib/apt/lists/*
+RUN apt-get update && apt-get install -y --no-install-recommends python3 ca-certificates git && rm -rf /var/lib/apt/lists/*
COPY . /src/
RUN python3 /src/packaging/package_terminal.py --out /out --cache /cache \
&& mkdir -p /opt/strata \
diff --git a/docs/DOCKER.md b/docs/DOCKER.md
index f4b5ee62c..7b22aa846 100644
--- a/docs/DOCKER.md
+++ b/docs/DOCKER.md
@@ -297,7 +297,10 @@ library inside the finished image. A package that fails any of these stops the b
`docker/smoke-test.sh` runs the checks we ran on real cards (serves, key, chat, tool call, monitor, clean stop, memory freed, restart); see its first lines for the variables.
**From source.** `docker build -t strata:src .` in the repository compiles the CUDA 13 engine inside Docker, makes the package
-and puts it in the same kind of image (`--build-arg CUDA_ARCHITECTURES=89` for one card only, much faster). For AMD and Intel
+and puts it in the same kind of image (`--build-arg CUDA_ARCHITECTURES=89` for one card only, much faster). For a GTX 10, P100 or
+V100 build the CUDA 12 variant: `docker build -t strata:src-cuda12 --build-arg CUDA_IMAGE=nvidia/cuda:12.9.1-devel-ubuntu24.04
+--build-arg CUDA_ARCHITECTURES="60;61;70;75;80;86;89" --build-arg BUILD_VISION=0 .` (the build turns on the Pascal / Volta code by itself
+when an architecture below 75 is listed, and names the package `cuda12` with driver 575.51). For AMD and Intel
builds from source see [AMD_HIP.md](AMD_HIP.md) and [INTEL.md](INTEL.md); then package the result with
`packaging/package_engine.py` and use `docker/build.sh`.
diff --git a/packaging/ENGINE-COMPONENT.md b/packaging/ENGINE-COMPONENT.md
index db7c12215..3a633e0d2 100644
--- a/packaging/ENGINE-COMPONENT.md
+++ b/packaging/ENGINE-COMPONENT.md
@@ -99,7 +99,8 @@ the manifest file (which holds every file's hash), used as the package's digest
| `windows-x86_64` cuda, variant `cuda13` | the owner PC, the same cmake line as the release zips (CUDA 13.0, `75;86;89;120` real + `120` PTX, `STRATA_PORTABLE=ON`) | `strata.exe`, `strata_kq.dll`, `strata-vision.exe` (reused, unchanged since 0.1.44), `BUILD.json` | `min_driver` nvidia 580.65 |
| `windows-x86_64` cuda, variant `cuda12` | the owner PC (CUDA 12.9, `60;61;70;75;80;86;89` real + `89` PTX, `STRATA_EXPERIMENTAL_SM60=ON`) | the same four files | experimental, `min_driver` nvidia 575.51; the CUDA 13 build wins when both fit |
| `windows-x86_64` hip | the owner PC, `tools/hip/build_windows.bat` (ROCm 10.2 nightly wheels) then `tools/hip/package_windows.py` | `strata.exe`, `strata-device.exe`, the ROCm DLLs and the rocBLAS/hipBLASLt kernels of the zip (`--tree`) | no Windows AMD test machine yet; with `STRATA_NO_VISION` unset it also builds the two picture readers (`strata-vision.exe` on the CPU, `strata-vision-hip.exe` on the GPU) |
-| `linux-x86_64` cuda | a Linux box with the CUDA toolkit | `strata`, `libstrata_kq.so`, `libcudart/libcublas/libcublasLt` (13.x) | the GPU driver's `libcuda` stays the PC's |
+| `linux-x86_64` cuda, variant `cuda13` | a Linux box with the CUDA 13 toolkit, `VARIANT=cuda13 packaging/build_cuda_linux.sh` | `strata`, `libstrata_kq.so`, `libcudart/libcublas/libcublasLt` (13.x) | the GPU driver's `libcuda` stays the PC's |
+| `linux-x86_64` cuda, variant `cuda12` | a Linux box with CUDA 12.9, `VARIANT=cuda12 packaging/build_cuda_linux.sh` (`60;61;70;75;80;86;89` real + `89` PTX, `STRATA_EXPERIMENTAL_SM60=ON`) | the same files with the 12.x libraries | experimental, `min_driver` nvidia 575.51; the Pascal and Volta package (the 0.2.1 one lacked sm_60 / 61 / 70, #3503); `package_engine.py` refuses a cuda12 package without them |
| `linux-x86_64` hip | a box with ROCm (the Python wheels are enough) | `strata`, the HIP runtime, hipBLAS/hipBLASLt/rocBLAS, comgr + libLLVM, `rocblas/library` and `hipblaslt/library` for the listed gfx archs; `packaging/build_hip_linux.sh` builds it with both picture readers |
| `linux-x86_64` hip, variant `rx6000` (RX 6000, gfx1030 / gfx1031) | `VARIANT=rx6000 packaging/build_hip_linux.sh` in the jammy chroot, ROCm 7.13.0a20260515 `gfx103X-all` wheels | the same set as the other HIP packages, with `rocblas/library` for gfx1030 / gfx1031; no `hipblaslt/library` kernels exist for them and no `tuning/` table | built, not tested on a real RX 6000 card (the manifest notes say so). `windows-x86_64` hip, variant `rdna2`: the same archs from `tools/hip/build_windows.bat` with `STRATA_HIP_ARCHS=gfx1030;gfx1031`, packaged with `--variant rdna2`. A PC with an RX 7000 / 9000 card beside an RX 6000 one never takes it (`engine_component.hip_preference`) |
| `linux-x86_64` hip, gfx1103 (Radeon 780M / Phoenix iGPU) | `packaging/build_hip_gfx1103.sh` on the mini PC (TheRock gfx110X-all SDK) | the same set as the gfx1200/1201 package, with `rocblas/library` and `hipblaslt/library` for gfx1103 (600 MiB) | experimental, unvalidated: the iGPU resets now and then (MES REMOVE_QUEUE); never kill the engine mid-request; run configs set `STRATA_NO_ARENA_THP=1`. `engine_component.resolve` picks a HIP package only when its `archs` list a gfx target the Linux kfd topology reports |
diff --git a/packaging/build_cuda_linux.sh b/packaging/build_cuda_linux.sh
new file mode 100644
index 000000000..6c07a5ba0
--- /dev/null
+++ b/packaging/build_cuda_linux.sh
@@ -0,0 +1,53 @@
+#!/bin/sh
+# Build the Linux CUDA engine package for the release feed.
+#
+# VARIANT=cuda12 packaging/build_cuda_linux.sh # CUDA 12.9: GTX 10 (Pascal), Tesla P100, V100, and the newer cards up to sm_89
+# VARIANT=cuda13 packaging/build_cuda_linux.sh # CUDA 13: RTX 20 and newer (sm_75 .. sm_120)
+#
+# Run it on a Linux x86_64 box with the CUDA toolkit of that variant on PATH (nvcc), a Python 3 venv with requirements.txt and
+# requirements-build.txt, and the pinned llama.cpp in third_party/llama.cpp (python -c "import setup; setup.get_llama_cpp()").
+# The package lands in $OUT (default ./out-engine-<variant>). Nothing is uploaded.
+#
+# The cuda12 package is the one the feed and docs/DOCKER.md offer to Pascal and Volta cards, so it needs machine code for sm_60,
+# sm_61 and sm_70 and the -DSTRATA_EXPERIMENTAL_SM60=ON build (#3503: the 0.2.1 package had sm_75 / 86 / 89 only, and a P100
+# container restarted in a loop with "this PC's GPU is sm_60"). packaging/package_engine.py refuses a cuda12 package without them.
+set -eu
+variant=${VARIANT:-cuda13}
+repo=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd -P)
+case "$variant" in
+ cuda12) archs=${ARCHS:-60;61;70;75;80;86;89}; mindriver=575.51; sm60=ON ;;
+ cuda13) archs=${ARCHS:-75;86;89;120}; mindriver=580.65; sm60=OFF ;;
+ *) echo "build_cuda_linux.sh: VARIANT must be cuda12 or cuda13" >&2; exit 2 ;;
+esac
+out=${OUT:-$repo/out-engine-$variant}
+build=${BUILD:-$repo/build-$variant}
+py=${PYTHON:-python3}
+cudafull=$(nvcc --version | sed -n 's/.*release \([0-9][0-9]*\.[0-9][0-9]*\).*/\1/p' | head -1)
+case "$variant:$cudafull" in
+ cuda12:12.*|cuda13:13.*) ;;
+ *) echo "build_cuda_linux.sh: $variant needs the matching CUDA toolkit; nvcc here is ${cudafull:-missing}" >&2; exit 2 ;;
+esac
+
+real=""; for a in $(echo "$archs" | tr ';,' ' '); do real="$real;${a}-real"; done; real=${real#;}
+last=$(echo "$archs" | tr ';,' ' ' | awk '{print $NF}')
+cd "$repo"
+cmake -G Ninja -S . -B "$build" -DCMAKE_BUILD_TYPE=Release -DSTRATA_ENABLE_CUDA=ON -DSTRATA_BUILD_TESTS=OFF \
+ -DSTRATA_EXPERIMENTAL_SM60=$sm60 -DSTRATA_NATIVE_EXPERTS=ON -DSTRATA_PORTABLE=ON -DSTRATA_Q6K_EXPERTS=ON -DSTRATA_MMQ_KQUANTS=ON \
+ "-DCMAKE_CUDA_ARCHITECTURES=${real};${last}-virtual" -DSTRATA_GGML_DIR="$repo/third_party/llama.cpp"
+cmake --build "$build" --target strata strata_kq -j "$(nproc)"
+
+ver=$(sed -n 's/^[[:space:]]*project([^)]*VERSION[[:space:]]*\([0-9][0-9.]*\).*/\1/p' CMakeLists.txt | head -1)
+stage=$(mktemp -d "${TMPDIR:-/tmp}/strata-engine-stage.XXXXXX")
+trap 'rm -rf "$stage"' EXIT
+cp "$build/strata" "$stage/"
+[ -f "$build/libstrata_kq.so" ] && cp "$build/libstrata_kq.so" "$stage/"
+echo "{\"version\": \"$ver\", \"source\": \"packaging/build_cuda_linux.sh\", \"archs\": \"$archs\", \"cuda\": \"$cudafull\", \"portable\": true}" > "$stage/BUILD.json"
+"$py" packaging/collect_libs.py --binary "$build/strata" --search /usr/local/cuda/lib64 --out "$stage" --also 'libcublasLt.so*'
+arch_args=""; for a in $(echo "$archs" | tr ';,' ' '); do arch_args="$arch_args --arch sm_$a"; done
+mkdir -p "$out"
+# shellcheck disable=SC2086
+"$py" packaging/package_engine.py --src "$stage" --backend cuda --version "$ver" --out "$out" $arch_args --arch "compute_$last" \
+ --min-driver nvidia=$mindriver --variant "$variant" --platform linux-x86_64 --build-id "release-${ver}-${variant}-${cudafull}" \
+ --include SYSTEM-LIBS.txt --include 'libstrata_kq*'
+for d in "$out"/*/; do "$py" packaging/package_engine.py --verify "$d"; "$py" packaging/binary_deps.py "$d"; done
+echo "built: $out"
diff --git a/packaging/package_engine.py b/packaging/package_engine.py
index 2c7c570a9..86e583681 100644
--- a/packaging/package_engine.py
+++ b/packaging/package_engine.py
@@ -95,6 +95,11 @@ def build(src: Path, backend: str, version: str, target: str, out: Path, *, entr
raise SystemExit(f"unknown backend {backend!r}")
if backend != "cpu" and not archs:
raise SystemExit(f"a {backend} package needs at least one --arch (sm_86, gfx1201, ...)")
+ if backend == "cuda" and variant == "cuda12" and not any(a in ("sm_60", "sm_61", "sm_70") for a in archs):
+ # #3503: the cuda12 package is what the feed and docs/DOCKER.md offer to GTX 10 / P100 / V100; one built for
+ # sm_75 and up starts a restart loop on those cards ("this PC's GPU is sm_60")
+ raise SystemExit("refusing to package the cuda12 engine without sm_60, sm_61 or sm_70: it is the Pascal and Volta build "
+ "(CUDA 12.9, -DSTRATA_EXPERIMENTAL_SM60=ON, --arch sm_60 --arch sm_61 --arch sm_70 ...; see packaging/build_cuda_linux.sh)")
if backend == "hip":
from strata_control import hipblaslt_tuning
problems = hipblaslt_tuning.check_stage(src, archs)
diff --git a/tests/terminal/test_engine_manifest.py b/tests/terminal/test_engine_manifest.py
index 2e388b499..a66529675 100644
--- a/tests/terminal/test_engine_manifest.py
+++ b/tests/terminal/test_engine_manifest.py
@@ -225,13 +225,22 @@ class Start(unittest.TestCase):
def test_two_builds_of_one_version_prefer_the_newer_driver_when_both_fit(self):
other = make_source(self.root / "src3")
- old = package_engine.build(other, "cuda", "0.1.45", HERE, self.out, archs=("sm_86",),
+ old = package_engine.build(other, "cuda", "0.1.45", HERE, self.out, archs=("sm_60", "sm_86"),
min_driver={"nvidia": "575.51"}, build_id="b-12", variant="cuda12")
self.assertEqual(old.name, f"0.1.45-{HERE}-cuda-cuda12")
self.assertEqual(ec.read_manifest(old).variant, "cuda12")
self.assertEqual(Path(self.start(driver="610.0")).parent.resolve(), self.folder.resolve()) # 580.65 build wins
self.assertEqual(Path(self.start(driver="576.0")).parent.resolve(), old.resolve()) # only the CUDA 12 one fits
+ def test_a_cuda12_package_without_pascal_or_volta_code_is_refused(self):
+ """#3503: the Linux cuda12 package shipped with 0.2.1 had sm_75/86/89 only; P100 containers restarted in a loop."""
+ other = make_source(self.root / "src4")
+ with self.assertRaises(SystemExit) as caught:
+ package_engine.build(other, "cuda", "0.1.45", HERE, self.out, archs=("sm_75", "sm_86", "sm_89", "compute_89"),
+ min_driver={"nvidia": "575.51"}, build_id="b-12", variant="cuda12")
+ self.assertIn("sm_60", str(caught.exception))
+ package_engine.build(other, "cuda", "0.1.45", HERE, self.out, archs=("sm_70", "sm_86"), build_id="b-13", variant="cuda12")
+
def test_a_good_package_does_not_hide_a_tampered_one(self):
newer = package_engine.build(make_source(self.root / "src2"), "cuda", "0.1.46", HERE, self.out,
archs=("sm_86",), build_id="b2")