Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
Fix #3503: the Linux cuda12 engine package carries sm_60/61/70; the source Dockerfile builds it
Niko1221366c8dbaaef6
diff --git a/Dockerfile b/Dockerfileindex 33cf7679d..f6560ee17 100644--- a/Dockerfile+++ b/Dockerfile@@ -5,6 +5,8 @@## docker build -t strata:src . # RTX 20/30/40/50 (compiles for each, ~25 min)# docker build -t strata:src --build-arg CUDA_ARCHITECTURES=86 . # one card only (faster); 86 = RTX 30, 89 = RTX 40, 120 = RTX 50+# docker build -t strata:src-cuda12 --build-arg CUDA_IMAGE=nvidia/cuda:12.9.1-devel-ubuntu24.04 \+# --build-arg CUDA_ARCHITECTURES="60;61;70;75;80;86;89" --build-arg BUILD_VISION=0 . # GTX 10 / P100 / V100 (driver 575+)# docker run --rm --gpus all -p 127.0.0.1:8080:8080 --ulimit memlock=-1 --shm-size 2g --stop-timeout 120 \# -v strata-data:/data -e MODEL=IQ3_XXS -e API_KEY=change-me-please-123 strata:src#@@ -35,7 +37,9 @@ RUN /venv/bin/python -c "import setup; print(setup.get_llama_cpp())"RUN set -e; \real=""; for a in $(echo "$CUDA_ARCHITECTURES" | tr ';,' ' '); do real="$real;${a}-real"; done; real=${real#;}; \last=$(echo "$CUDA_ARCHITECTURES" | tr ';,' ' ' | awk '{print $NF}'); \+ sm60=OFF; for a in $(echo "$CUDA_ARCHITECTURES" | tr ';,' ' '); do if [ "$a" -lt 75 ]; then sm60=ON; fi; done; \/venv/bin/cmake -G Ninja -S . -B build -DCMAKE_BUILD_TYPE=Release -DSTRATA_ENABLE_CUDA=ON -DSTRATA_BUILD_TESTS=OFF \+ -DSTRATA_EXPERIMENTAL_SM60=$sm60 \-DSTRATA_NATIVE_EXPERTS=ON -DSTRATA_PORTABLE=ON -DSTRATA_Q6K_EXPERTS=ON -DSTRATA_MMQ_KQUANTS=ON \"-DCMAKE_CUDA_ARCHITECTURES=${real};${last}-virtual" -DSTRATA_GGML_DIR=/src/third_party/llama.cpp; \/venv/bin/cmake --build build --target strata strata_kq -j "$(nproc)"; \@@ -50,19 +54,21 @@ RUN set -e; \[ -f build/libstrata_kq.so ] && cp build/libstrata_kq.so /stage/ || true; \[ -f build-vision/bin/strata-vision ] && cp build-vision/bin/strata-vision /stage/ || true; \ver="${ENGINE_VERSION:-$(sed -n 's/^[[:space:]]*project([^)]*VERSION[[:space:]]*\([0-9][0-9.]*\).*/\1/p' CMakeLists.txt | head -1)}"; \- echo "{\"version\": \"$ver\", \"source\": \"docker build\", \"archs\": \"$CUDA_ARCHITECTURES\", \"cuda\": \"13.0\", \"portable\": true}" > /stage/BUILD.json; \+ cudafull=$(nvcc --version | sed -n 's/.*release \([0-9][0-9]*\.[0-9][0-9]*\).*/\1/p' | head -1); cudamajor=${cudafull%%.*}; \+ if [ "$cudamajor" = 13 ]; then mindriver=580.65; else mindriver=575.51; fi; \+ echo "{\"version\": \"$ver\", \"source\": \"docker build\", \"archs\": \"$CUDA_ARCHITECTURES\", \"cuda\": \"$cudafull\", \"portable\": true}" > /stage/BUILD.json; \python3 packaging/collect_libs.py --binary build/strata --search /usr/local/cuda/lib64 --out /stage --also 'libcublasLt.so*'; \archs=""; for a in $(echo "$CUDA_ARCHITECTURES" | tr ';,' ' '); do archs="$archs --arch sm_$a"; done; \last=$(echo "$CUDA_ARCHITECTURES" | tr ';,' ' ' | awk '{print $NF}'); \python3 packaging/package_engine.py --src /stage --backend cuda --version "$ver" --out /engine $archs --arch "compute_$last" \- --min-driver nvidia=580.65 --variant cuda13 --platform linux-x86_64 --build-id "docker-source-${ver}-cuda13.0" --include SYSTEM-LIBS.txt --include 'libstrata_kq*' \+ --min-driver nvidia=$mindriver --variant cuda$cudamajor --platform linux-x86_64 --build-id "docker-source-${ver}-cuda${cudafull}" --include SYSTEM-LIBS.txt --include 'libstrata_kq*' \--include strata-vision; \for d in /engine/*/; do python3 packaging/package_engine.py --verify "$d"; python3 packaging/binary_deps.py "$d"; done# ---- 2. the Terminal (the release archive, built here)FROM ${RUNTIME_BASE} AS terminalENV DEBIAN_FRONTEND=noninteractive-RUN apt-get update && apt-get install -y --no-install-recommends python3 ca-certificates && rm -rf /var/lib/apt/lists/*+RUN apt-get update && apt-get install -y --no-install-recommends python3 ca-certificates git && rm -rf /var/lib/apt/lists/*COPY . /src/RUN python3 /src/packaging/package_terminal.py --out /out --cache /cache \&& mkdir -p /opt/strata \diff --git a/docs/DOCKER.md b/docs/DOCKER.mdindex f4b5ee62c..7b22aa846 100644--- a/docs/DOCKER.md+++ b/docs/DOCKER.md@@ -297,7 +297,10 @@ library inside the finished image. A package that fails any of these stops the b`docker/smoke-test.sh` runs the checks we ran on real cards (serves, key, chat, tool call, monitor, clean stop, memory freed, restart); see its first lines for the variables.**From source.** `docker build -t strata:src .` in the repository compiles the CUDA 13 engine inside Docker, makes the package-and puts it in the same kind of image (`--build-arg CUDA_ARCHITECTURES=89` for one card only, much faster). For AMD and Intel+and puts it in the same kind of image (`--build-arg CUDA_ARCHITECTURES=89` for one card only, much faster). For a GTX 10, P100 or+V100 build the CUDA 12 variant: `docker build -t strata:src-cuda12 --build-arg CUDA_IMAGE=nvidia/cuda:12.9.1-devel-ubuntu24.04+--build-arg CUDA_ARCHITECTURES="60;61;70;75;80;86;89" --build-arg BUILD_VISION=0 .` (the build turns on the Pascal / Volta code by itself+when an architecture below 75 is listed, and names the package `cuda12` with driver 575.51). For AMD and Intelbuilds from source see [AMD_HIP.md](AMD_HIP.md) and [INTEL.md](INTEL.md); then package the result with`packaging/package_engine.py` and use `docker/build.sh`.diff --git a/packaging/ENGINE-COMPONENT.md b/packaging/ENGINE-COMPONENT.mdindex db7c12215..3a633e0d2 100644--- a/packaging/ENGINE-COMPONENT.md+++ b/packaging/ENGINE-COMPONENT.md@@ -99,7 +99,8 @@ the manifest file (which holds every file's hash), used as the package's digest| `windows-x86_64` cuda, variant `cuda13` | the owner PC, the same cmake line as the release zips (CUDA 13.0, `75;86;89;120` real + `120` PTX, `STRATA_PORTABLE=ON`) | `strata.exe`, `strata_kq.dll`, `strata-vision.exe` (reused, unchanged since 0.1.44), `BUILD.json` | `min_driver` nvidia 580.65 || `windows-x86_64` cuda, variant `cuda12` | the owner PC (CUDA 12.9, `60;61;70;75;80;86;89` real + `89` PTX, `STRATA_EXPERIMENTAL_SM60=ON`) | the same four files | experimental, `min_driver` nvidia 575.51; the CUDA 13 build wins when both fit || `windows-x86_64` hip | the owner PC, `tools/hip/build_windows.bat` (ROCm 10.2 nightly wheels) then `tools/hip/package_windows.py` | `strata.exe`, `strata-device.exe`, the ROCm DLLs and the rocBLAS/hipBLASLt kernels of the zip (`--tree`) | no Windows AMD test machine yet; with `STRATA_NO_VISION` unset it also builds the two picture readers (`strata-vision.exe` on the CPU, `strata-vision-hip.exe` on the GPU) |-| `linux-x86_64` cuda | a Linux box with the CUDA toolkit | `strata`, `libstrata_kq.so`, `libcudart/libcublas/libcublasLt` (13.x) | the GPU driver's `libcuda` stays the PC's |+| `linux-x86_64` cuda, variant `cuda13` | a Linux box with the CUDA 13 toolkit, `VARIANT=cuda13 packaging/build_cuda_linux.sh` | `strata`, `libstrata_kq.so`, `libcudart/libcublas/libcublasLt` (13.x) | the GPU driver's `libcuda` stays the PC's |+| `linux-x86_64` cuda, variant `cuda12` | a Linux box with CUDA 12.9, `VARIANT=cuda12 packaging/build_cuda_linux.sh` (`60;61;70;75;80;86;89` real + `89` PTX, `STRATA_EXPERIMENTAL_SM60=ON`) | the same files with the 12.x libraries | experimental, `min_driver` nvidia 575.51; the Pascal and Volta package (the 0.2.1 one lacked sm_60 / 61 / 70, #3503); `package_engine.py` refuses a cuda12 package without them || `linux-x86_64` hip | a box with ROCm (the Python wheels are enough) | `strata`, the HIP runtime, hipBLAS/hipBLASLt/rocBLAS, comgr + libLLVM, `rocblas/library` and `hipblaslt/library` for the listed gfx archs; `packaging/build_hip_linux.sh` builds it with both picture readers || `linux-x86_64` hip, variant `rx6000` (RX 6000, gfx1030 / gfx1031) | `VARIANT=rx6000 packaging/build_hip_linux.sh` in the jammy chroot, ROCm 7.13.0a20260515 `gfx103X-all` wheels | the same set as the other HIP packages, with `rocblas/library` for gfx1030 / gfx1031; no `hipblaslt/library` kernels exist for them and no `tuning/` table | built, not tested on a real RX 6000 card (the manifest notes say so). `windows-x86_64` hip, variant `rdna2`: the same archs from `tools/hip/build_windows.bat` with `STRATA_HIP_ARCHS=gfx1030;gfx1031`, packaged with `--variant rdna2`. A PC with an RX 7000 / 9000 card beside an RX 6000 one never takes it (`engine_component.hip_preference`) || `linux-x86_64` hip, gfx1103 (Radeon 780M / Phoenix iGPU) | `packaging/build_hip_gfx1103.sh` on the mini PC (TheRock gfx110X-all SDK) | the same set as the gfx1200/1201 package, with `rocblas/library` and `hipblaslt/library` for gfx1103 (600 MiB) | experimental, unvalidated: the iGPU resets now and then (MES REMOVE_QUEUE); never kill the engine mid-request; run configs set `STRATA_NO_ARENA_THP=1`. `engine_component.resolve` picks a HIP package only when its `archs` list a gfx target the Linux kfd topology reports |diff --git a/packaging/build_cuda_linux.sh b/packaging/build_cuda_linux.shnew file mode 100644index 000000000..6c07a5ba0--- /dev/null+++ b/packaging/build_cuda_linux.sh@@ -0,0 +1,53 @@+#!/bin/sh+# Build the Linux CUDA engine package for the release feed.+#+# VARIANT=cuda12 packaging/build_cuda_linux.sh # CUDA 12.9: GTX 10 (Pascal), Tesla P100, V100, and the newer cards up to sm_89+# VARIANT=cuda13 packaging/build_cuda_linux.sh # CUDA 13: RTX 20 and newer (sm_75 .. sm_120)+#+# Run it on a Linux x86_64 box with the CUDA toolkit of that variant on PATH (nvcc), a Python 3 venv with requirements.txt and+# requirements-build.txt, and the pinned llama.cpp in third_party/llama.cpp (python -c "import setup; setup.get_llama_cpp()").+# The package lands in $OUT (default ./out-engine-<variant>). Nothing is uploaded.+#+# The cuda12 package is the one the feed and docs/DOCKER.md offer to Pascal and Volta cards, so it needs machine code for sm_60,+# sm_61 and sm_70 and the -DSTRATA_EXPERIMENTAL_SM60=ON build (#3503: the 0.2.1 package had sm_75 / 86 / 89 only, and a P100+# container restarted in a loop with "this PC's GPU is sm_60"). packaging/package_engine.py refuses a cuda12 package without them.+set -eu+variant=${VARIANT:-cuda13}+repo=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd -P)+case "$variant" in+ cuda12) archs=${ARCHS:-60;61;70;75;80;86;89}; mindriver=575.51; sm60=ON ;;+ cuda13) archs=${ARCHS:-75;86;89;120}; mindriver=580.65; sm60=OFF ;;+ *) echo "build_cuda_linux.sh: VARIANT must be cuda12 or cuda13" >&2; exit 2 ;;+esac+out=${OUT:-$repo/out-engine-$variant}+build=${BUILD:-$repo/build-$variant}+py=${PYTHON:-python3}+cudafull=$(nvcc --version | sed -n 's/.*release \([0-9][0-9]*\.[0-9][0-9]*\).*/\1/p' | head -1)+case "$variant:$cudafull" in+ cuda12:12.*|cuda13:13.*) ;;+ *) echo "build_cuda_linux.sh: $variant needs the matching CUDA toolkit; nvcc here is ${cudafull:-missing}" >&2; exit 2 ;;+esac++real=""; for a in $(echo "$archs" | tr ';,' ' '); do real="$real;${a}-real"; done; real=${real#;}+last=$(echo "$archs" | tr ';,' ' ' | awk '{print $NF}')+cd "$repo"+cmake -G Ninja -S . -B "$build" -DCMAKE_BUILD_TYPE=Release -DSTRATA_ENABLE_CUDA=ON -DSTRATA_BUILD_TESTS=OFF \+ -DSTRATA_EXPERIMENTAL_SM60=$sm60 -DSTRATA_NATIVE_EXPERTS=ON -DSTRATA_PORTABLE=ON -DSTRATA_Q6K_EXPERTS=ON -DSTRATA_MMQ_KQUANTS=ON \+ "-DCMAKE_CUDA_ARCHITECTURES=${real};${last}-virtual" -DSTRATA_GGML_DIR="$repo/third_party/llama.cpp"+cmake --build "$build" --target strata strata_kq -j "$(nproc)"++ver=$(sed -n 's/^[[:space:]]*project([^)]*VERSION[[:space:]]*\([0-9][0-9.]*\).*/\1/p' CMakeLists.txt | head -1)+stage=$(mktemp -d "${TMPDIR:-/tmp}/strata-engine-stage.XXXXXX")+trap 'rm -rf "$stage"' EXIT+cp "$build/strata" "$stage/"+[ -f "$build/libstrata_kq.so" ] && cp "$build/libstrata_kq.so" "$stage/"+echo "{\"version\": \"$ver\", \"source\": \"packaging/build_cuda_linux.sh\", \"archs\": \"$archs\", \"cuda\": \"$cudafull\", \"portable\": true}" > "$stage/BUILD.json"+"$py" packaging/collect_libs.py --binary "$build/strata" --search /usr/local/cuda/lib64 --out "$stage" --also 'libcublasLt.so*'+arch_args=""; for a in $(echo "$archs" | tr ';,' ' '); do arch_args="$arch_args --arch sm_$a"; done+mkdir -p "$out"+# shellcheck disable=SC2086+"$py" packaging/package_engine.py --src "$stage" --backend cuda --version "$ver" --out "$out" $arch_args --arch "compute_$last" \+ --min-driver nvidia=$mindriver --variant "$variant" --platform linux-x86_64 --build-id "release-${ver}-${variant}-${cudafull}" \+ --include SYSTEM-LIBS.txt --include 'libstrata_kq*'+for d in "$out"/*/; do "$py" packaging/package_engine.py --verify "$d"; "$py" packaging/binary_deps.py "$d"; done+echo "built: $out"diff --git a/packaging/package_engine.py b/packaging/package_engine.pyindex 2c7c570a9..86e583681 100644--- a/packaging/package_engine.py+++ b/packaging/package_engine.py@@ -95,6 +95,11 @@ def build(src: Path, backend: str, version: str, target: str, out: Path, *, entrraise SystemExit(f"unknown backend {backend!r}")if backend != "cpu" and not archs:raise SystemExit(f"a {backend} package needs at least one --arch (sm_86, gfx1201, ...)")+ if backend == "cuda" and variant == "cuda12" and not any(a in ("sm_60", "sm_61", "sm_70") for a in archs):+ # #3503: the cuda12 package is what the feed and docs/DOCKER.md offer to GTX 10 / P100 / V100; one built for+ # sm_75 and up starts a restart loop on those cards ("this PC's GPU is sm_60")+ raise SystemExit("refusing to package the cuda12 engine without sm_60, sm_61 or sm_70: it is the Pascal and Volta build "+ "(CUDA 12.9, -DSTRATA_EXPERIMENTAL_SM60=ON, --arch sm_60 --arch sm_61 --arch sm_70 ...; see packaging/build_cuda_linux.sh)")if backend == "hip":from strata_control import hipblaslt_tuningproblems = hipblaslt_tuning.check_stage(src, archs)diff --git a/tests/terminal/test_engine_manifest.py b/tests/terminal/test_engine_manifest.pyindex 2e388b499..a66529675 100644--- a/tests/terminal/test_engine_manifest.py+++ b/tests/terminal/test_engine_manifest.py@@ -225,13 +225,22 @@ class Start(unittest.TestCase):def test_two_builds_of_one_version_prefer_the_newer_driver_when_both_fit(self):other = make_source(self.root / "src3")- old = package_engine.build(other, "cuda", "0.1.45", HERE, self.out, archs=("sm_86",),+ old = package_engine.build(other, "cuda", "0.1.45", HERE, self.out, archs=("sm_60", "sm_86"),min_driver={"nvidia": "575.51"}, build_id="b-12", variant="cuda12")self.assertEqual(old.name, f"0.1.45-{HERE}-cuda-cuda12")self.assertEqual(ec.read_manifest(old).variant, "cuda12")self.assertEqual(Path(self.start(driver="610.0")).parent.resolve(), self.folder.resolve()) # 580.65 build winsself.assertEqual(Path(self.start(driver="576.0")).parent.resolve(), old.resolve()) # only the CUDA 12 one fits+ def test_a_cuda12_package_without_pascal_or_volta_code_is_refused(self):+ """#3503: the Linux cuda12 package shipped with 0.2.1 had sm_75/86/89 only; P100 containers restarted in a loop."""+ other = make_source(self.root / "src4")+ with self.assertRaises(SystemExit) as caught:+ package_engine.build(other, "cuda", "0.1.45", HERE, self.out, archs=("sm_75", "sm_86", "sm_89", "compute_89"),+ min_driver={"nvidia": "575.51"}, build_id="b-12", variant="cuda12")+ self.assertIn("sm_60", str(caught.exception))+ package_engine.build(other, "cuda", "0.1.45", HERE, self.out, archs=("sm_70", "sm_86"), build_id="b-13", variant="cuda12")+def test_a_good_package_does_not_hide_a_tampered_one(self):newer = package_engine.build(make_source(self.root / "src2"), "cuda", "0.1.46", HERE, self.out,archs=("sm_86",), build_id="b2")