Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
Docker: GPU/GPUS take effect on a volume that already has the model, tuning wording, build id of the source build
Niko1221a00ed935f1f7
diff --git a/Dockerfile b/Dockerfileindex 29f96a449..33cf7679d 100644--- a/Dockerfile+++ b/Dockerfile@@ -55,7 +55,7 @@ RUN set -e; \archs=""; for a in $(echo "$CUDA_ARCHITECTURES" | tr ';,' ' '); do archs="$archs --arch sm_$a"; done; \last=$(echo "$CUDA_ARCHITECTURES" | tr ';,' ' ' | awk '{print $NF}'); \python3 packaging/package_engine.py --src /stage --backend cuda --version "$ver" --out /engine $archs --arch "compute_$last" \- --min-driver nvidia=580.65 --variant cuda13 --platform linux-x86_64 --include SYSTEM-LIBS.txt --include 'libstrata_kq*' \+ --min-driver nvidia=580.65 --variant cuda13 --platform linux-x86_64 --build-id "docker-source-${ver}-cuda13.0" --include SYSTEM-LIBS.txt --include 'libstrata_kq*' \--include strata-vision; \for d in /engine/*/; do python3 packaging/package_engine.py --verify "$d"; python3 packaging/binary_deps.py "$d"; donediff --git a/docker/compose/docker-compose.nvidia.yml b/docker/compose/docker-compose.nvidia.ymlindex f55fb2e0a..c14d349e5 100644--- a/docker/compose/docker-compose.nvidia.yml+++ b/docker/compose/docker-compose.nvidia.yml@@ -19,7 +19,7 @@ services:CONTEXT: ${CONTEXT:-32k}PARALLEL: ${PARALLEL:-} # empty: fit to the cardVISION: ${VISION:-off} # on: the model reads pictures (about 0.9 GB more to download)- TUNE: ${TUNE:-skip} # quick: about 20 minutes once, then kept+ TUNE: ${TUNE:-skip} # quick: up to about 20 minutes, once, then keptAPI_KEY: ${API_KEY:-} # empty: made on the first start; docker exec strata strata launch --show-key# GPUS: "0,1" # several cards: which onesvolumes:diff --git a/docker/entrypoint.sh b/docker/entrypoint.shindex 23e186f0a..3d7341b65 100644--- a/docker/entrypoint.sh+++ b/docker/entrypoint.sh@@ -25,6 +25,7 @@ set -uAPP=/opt/strataSTRATA="$APP/terminal/bin/strata"+PYBIN="$APP/terminal/python/bin/python3"; [ -x "$PYBIN" ] || PYBIN=$(command -v python3 || echo python3)STATE_DIR="${XDG_STATE_HOME:-/data/state}/strata"KEY_FILE="${STRATA_KEY_FILE:-/data/state/api-key}"PHASE_FILE=/run/strata-phase@@ -120,7 +121,20 @@ if [ -n "${VISION:-}" ]; thenset -- "$@" --vision "$v"; have=1figpus="${GPUS:-${GPU:-}}"-case "$gpus" in ""|all|ALL) ;; *) set -- "$@" --gpu "$gpus"; have=1 ;; esac+case "$gpus" in ""|all|ALL) ;; *)+ set -- "$@" --gpu "$gpus"; have=1+ # `launch --gpu` is applied when a model is first set up; a model that is already set up keeps its card. So that GPU / GPUS take effect on+ # a volume that already has the model, run settings that name another card are dropped: setup writes them again from the files that are there.+ want=$(printf '%s' "$gpus" | tr -d ' ')+ for c in /data/install/strata-*.json; do+ [ -f "$c" ] || continue+ cur=$("$PYBIN" -c 'import json, sys; g = json.load(open(sys.argv[1])).get("gpu"); print(",".join(map(str, g)) if isinstance(g, list) else ("" if g is None else g))' "$c" 2>/dev/null)+ if [ "$cur" != "$want" ]; then+ say "The saved run settings use graphics card '${cur:-default}', GPU/GPUS asks for '$want': making them again (model files are kept)."+ rm -f "$c"+ fi+ done ;;+esac[ -n "${STRATA_SET:-}" ] && { for kv in $STRATA_SET; do set -- "$@" --set "$kv"; done; have=1; }[ -n "$key" ] && { set -- "$@" --api-key-env STRATA_LAUNCH_KEY; }@@ -133,7 +147,7 @@ if [ -n "$tune_mode" ] && [ "$tune_mode" != "skip" ]; thensay "This preset is already tuned; keeping the tuned settings (set RETUNE=1 to tune again)."elseset -- "$@" --tune "$tune_mode"; have=1- say "Tuning is on (TUNE=$tune_mode): the first start takes about $([ "$tune_mode" = quick ] && echo 20 || echo 120) minutes longer."+ say "Tuning is on (TUNE=$tune_mode): the first start takes longer (up to about $([ "$tune_mode" = quick ] && echo 20 || echo 120) minutes), once."fifi[ "${TUNE_ALLOW_SMALL:-0}" = "1" ] && set -- "$@" --tune-allow-small@@ -167,7 +181,6 @@ trap on_term TERM INT# A command started with & by a non-interactive sh ignores SIGINT, so `kill -INT` would do nothing. The launch must see it (it is how it# stops a download or a load cleanly), so it is started through a one-line Python that puts SIGINT back before it runs the Terminal.-PYBIN="$APP/terminal/python/bin/python3"; [ -x "$PYBIN" ] || PYBIN=$(command -v python3 || echo python3)RESET_INT='import os, signal, sys; signal.signal(signal.SIGINT, signal.SIG_DFL); os.execv(sys.argv[1], sys.argv[1:])'phase preparingdiff --git a/docs/DOCKER.md b/docs/DOCKER.mdindex 6a519b93a..94896d728 100644--- a/docs/DOCKER.md+++ b/docs/DOCKER.md@@ -115,7 +115,7 @@ screens; leave out what you do not care about.| `VISION` | `off`, `on` (the model reads pictures), `gpu`, `cpu`; the picture-reader file (about 0.9 GB) is downloaded with the model | `off` || `KV` | conversation memory format: `int8`, `q4_0` (half the memory), `k8v4`, `fp16` | `int8` || `GPU` / `GPUS` | card numbers, `0` or `0,1`; empty or `all` = the engine's choice | all that fit |-| `TUNE` | `quick` (about 20 minutes, once), `thorough`, `skip`; done once and kept in the preset (`RETUNE=1` tunes again) | `skip` |+| `TUNE` | `quick` (up to about 20 minutes, once), `thorough`, `skip`; done once and kept in the preset (`RETUNE=1` tunes again) | `skip` || `API_KEY` / `API_KEY_FILE` | the key clients must send ([below](#the-api-key)); the file form is for Docker secrets | made for you || `PORT` | the port inside the container | `8080` || `PRESET` | the preset name in `/data` | `docker` |diff --git a/tools/test_docker_entrypoint.py b/tools/test_docker_entrypoint.pyindex fe095b1c7..cb76c85dd 100644--- a/tools/test_docker_entrypoint.py+++ b/tools/test_docker_entrypoint.py@@ -144,6 +144,28 @@ class Entrypoint(unittest.TestCase):self.assertIn("API_KEY is short", err)self.assertIn("HOST is from the older image", err)+ def test_a_changed_card_drops_the_saved_run_settings_of_the_old_card(self):+ install = self.tmp / "data" / "install"+ install.mkdir(parents=True)+ (install / "strata-q2_0.json").write_text('{"gpu": 3, "args": []}')+ (install / "strata-iq3_s.json").write_text('{"gpu": 1, "args": []}')+ p = self.start(MODEL="Q2_0", GPU="1")+ line = self.launch_line(self.wait_for("--show-key"))+ self.assertIn("--gpu 1", line)+ code, out, err = self.stop(p)+ self.assertIn("making them again", out)+ self.assertFalse((install / "strata-q2_0.json").exists()) # card 3 -> 1: made again from the files+ self.assertTrue((install / "strata-iq3_s.json").exists()) # already card 1: kept++ def test_all_cards_leaves_the_saved_run_settings_alone(self):+ install = self.tmp / "data" / "install"+ install.mkdir(parents=True)+ (install / "strata-q2_0.json").write_text('{"gpu": 3, "args": []}')+ p = self.start(MODEL="Q2_0", GPUS="all")+ self.wait_for("--show-key")+ self.stop(p)+ self.assertTrue((install / "strata-q2_0.json").exists())+def test_other_commands_run_instead_of_the_server(self):r = subprocess.run([SH, str(self.script), "strata", "launch", "--plan", "--model", "q2_0"], env=self.env(), capture_output=True,text=True, timeout=30)