Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
serve, launch: "pool_cores" in the run config and strata launch --pool-cores
Niko12212aa1ec2d56cd
diff --git a/docs/TERMINAL_LAUNCH.md b/docs/TERMINAL_LAUNCH.mdindex 23444e1d1..871bcffb9 100644--- a/docs/TERMINAL_LAUNCH.md+++ b/docs/TERMINAL_LAUNCH.md@@ -62,6 +62,7 @@ strata launch --model iq3_xxs --context 128k --parallel 4 --vision off --tune qu--access lan --port 8080 --save-preset "Office" --no-tui # serve and print the addressstrata launch --model-path /data/models/IQ3_XXS --no-tui # files you already havestrata launch --preset Office --set prefill=65536 --set STRATA_PIN_BY_NEED=1 --no-tui+strata launch --model q2_0 --pool-cores 2-14:2 --no-tui # experimental: pin the CPU workers to these logical processors (see DETAILS.md)strata launch --plan --model q2_0 # what would happen; nothing changesstrata launch --model q2_0 --yes --no-tui # no questions (for one: download on a computer with no engine yet)strata launch --model q2_0 --open # also open the server's page in your browser once it servesdiff --git a/serve/server.py b/serve/server.pyindex 1cfadc6af..bf335f9ec 100644--- a/serve/server.py+++ b/serve/server.py@@ -2948,6 +2948,7 @@ def engine_args(cfg: dict) -> list[str]:if isinstance(seg, int) and not isinstance(seg, bool) and seg > 0 and "--vram-segment-mib" not in args:args += ["--vram-segment-mib", str(seg)]args += parallel_args(cfg, args)+ args += pool_cores_args(cfg, args)return learned_profile_args(cfg, args)@@ -2975,6 +2976,32 @@ def parallel_args(cfg: dict, args: list[str]) -> list[str]:return ["--batch", str(n)]+_POOL_CORES_OK = re.compile(r"^[0-9]+(-[0-9]+(:[0-9]+)?)?(,[0-9]+(-[0-9]+(:[0-9]+)?)?)*$")+++def pool_cores_args(cfg: dict, args: list[str]) -> list[str]:+ """Experimental, off by default: "pool_cores": "0,2,4,6" (or "2-14:2", or a list of numbers) in the config pins the CPU+ expert workers to exactly those logical processors (--pool-cores; docs/DETAILS.md, "Which logical processors the CPU+ workers use"). The engine warns about two entries on one core or the host's own and leaves out processors the process may+ not use; a text this cannot read is said here and ignored (recommend, never force). An engine option already in the args wins."""+ value = cfg.get("pool_cores")+ if value is None or value is False or value == "" or value == []:+ return []+ if "--pool-cores" in args:+ return []+ if isinstance(value, list) and value and all(isinstance(v, int) and not isinstance(v, bool) and v >= 0 for v in value):+ text = ",".join(str(v) for v in value)+ elif isinstance(value, str):+ text = value.replace(" ", "")+ else:+ text = ""+ if not _POOL_CORES_OK.match(text):+ print(f'[strata] "pool_cores" must be logical processor numbers like "0,2,4,6" or "2-14:2", not {value!r}: ignored '+ "(the CPU workers keep their default placement)", flush=True)+ return []+ return ["--pool-cores", text]++def profile_shape(path: str) -> tuple[int, int] | None:"""An expert profile's (layers, experts per layer), from its header (tools/make_profile.py's format), or Nonewhen the file is missing, is not one or is shorter than the pairs its header promises."""diff --git a/serve/test_pool_cores.py b/serve/test_pool_cores.pynew file mode 100644index 000000000..ca0c94ca9--- /dev/null+++ b/serve/test_pool_cores.py@@ -0,0 +1,41 @@+"""The run config's experimental "pool_cores" (the CPU expert workers' logical processors; --pool-cores in the engine)."""+import contextlib+import io+import unittest++from serve.server import engine_args, pool_cores_args+++class PoolCoresConfig(unittest.TestCase):+ def test_off_by_default(self):+ self.assertEqual(pool_cores_args({}, []), [])+ self.assertEqual(pool_cores_args({"pool_cores": None}, []), [])+ self.assertEqual(pool_cores_args({"pool_cores": ""}, []), [])+ self.assertEqual(pool_cores_args({"pool_cores": []}, []), [])+ self.assertEqual(pool_cores_args({"pool_cores": False}, []), [])+ self.assertEqual(engine_args({"args": ["--pack", "p"]}), ["--pack", "p"]) # nothing is added to an ordinary config++ def test_text_and_list_forms(self):+ self.assertEqual(pool_cores_args({"pool_cores": "2,4,6,8"}, []), ["--pool-cores", "2,4,6,8"])+ self.assertEqual(pool_cores_args({"pool_cores": " 2-14:2 "}, []), ["--pool-cores", "2-14:2"])+ self.assertEqual(pool_cores_args({"pool_cores": [2, 4, 6]}, []), ["--pool-cores", "2,4,6"])+ self.assertEqual(pool_cores_args({"pool_cores": "0,2-6,10"}, []), ["--pool-cores", "0,2-6,10"])++ def test_the_args_own_value_wins(self):+ self.assertEqual(pool_cores_args({"pool_cores": "2,4"}, ["--pool-cores", "6,8"]), [])++ def test_a_list_that_cannot_be_read_is_said_and_ignored(self):+ for bad in ("two,four", "1,,2", "6-2x", -1, True, 5, [1, "a"], [-1], {"a": 1}):+ buf = io.StringIO()+ with contextlib.redirect_stdout(buf):+ r = pool_cores_args({"pool_cores": bad}, [])+ self.assertEqual(r, [], bad)+ self.assertIn("pool_cores", buf.getvalue(), bad)++ def test_reaches_the_engine_arguments(self):+ args = engine_args({"args": ["--pack", "p"], "pool_cores": "2,4,6"})+ self.assertEqual(args[-2:], ["--pool-cores", "2,4,6"])+++if __name__ == "__main__":+ unittest.main()diff --git a/strata_cli/launch/cmd.py b/strata_cli/launch/cmd.pyindex bd401cd3f..c293b6b28 100644--- a/strata_cli/launch/cmd.py+++ b/strata_cli/launch/cmd.py@@ -5,6 +5,7 @@ from __future__ import annotationsimport argparseimport jsonimport os+import reimport sysimport threadingfrom pathlib import Path@@ -103,6 +104,10 @@ def add_parsers(sp: argparse._SubParsersAction) -> list[argparse.ArgumentParser]g.add_argument("--tune-allow-small", action="store_true", help="let the tuner also try settings that can change answers very slightly")g.add_argument("--set", action="append", default=[], metavar="KEY=VALUE",help="an advanced engine setting or STRATA_* variable, e.g. prefill=65536 (repeat); unknown names get a warning, not a refusal")+ g.add_argument("--pool-cores", metavar="LIST",+ help="experimental: pin the CPU workers to exactly these logical processors, e.g. 2,4,6,8 or 2-14:2 (the numbers Task Manager or "+ "lscpu show). The default is one worker per physical core, which the engine's start line names; "+ "two entries on one core are kept with a warning")a = p.add_argument_group("who can use it")a.add_argument("--access", type=_choice("a way to share Strata", S.ACCESS_KINDS), metavar="local|lan|cloudflare|tailscale",help="local: this computer only; lan: your network; cloudflare: a quick tunnel; tailscale: your tailnet")@@ -235,6 +240,13 @@ def settings_from_args(a: argparse.Namespace, catalog=None) -> tuple[S.LaunchSets.no_key = Falseif a.set:s.extra = {**s.extra, **S.settings_from_pairs(a.set)}+ if a.pool_cores:+ text = a.pool_cores.replace(" ", "")+ if not re.fullmatch(r"\d+(-\d+(:\d+)?)?(,\d+(-\d+(:\d+)?)?)*", text):+ notices.append(S.Notice("warn", f"'{a.pool_cores}' is not a list of logical processor numbers, so --pool-cores is left out.",+ "Write it like 2,4,6,8 or 2-14:2. The CPU workers keep their default placement."))+ else:+ s.extra = {**s.extra, "--pool-cores": text}if a.save_preset:s.name = a.save_presetreturn s, notices@@ -587,6 +599,7 @@ def run_launch(a: argparse.Namespace) -> int:except LaunchError as e:return fail(f"{e.text}" + (f" {e.next}" if e.next else ""), e.code)s, notes = apply_defaults(s, catalog)+ notes = [*notices, *notes] # what the flags themselves said (a --pool-cores list that could not be read, a key beside --no-api-key)errors = [n for n in notes if n.level == "error"]if errors:return fail(errors[0].line(), 2)diff --git a/tests/terminal/test_launch.py b/tests/terminal/test_launch.pyindex e471c9612..13ad5837c 100644--- a/tests/terminal/test_launch.py+++ b/tests/terminal/test_launch.py@@ -794,6 +794,20 @@ class FlagsCommand(unittest.TestCase):self.assertEqual(s["extra"], {"--prefill": "65536", "STRATA_PIN_BY_NEED": "1"})self.assertIn("--set prefill=65536", p["flags"])+ def test_pool_cores_is_an_experimental_advanced_setting(self):+ code, out, err = run_cli(["launch", "--help"])+ self.assertIn("--pool-cores", out)+ p = self.plan("--model", "q2_0", "--pool-cores", "2,4, 6,8")+ self.assertEqual(p["settings"]["extra"], {"--pool-cores": "2,4,6,8"})+ self.assertIn("--set pool-cores=2,4,6,8", p["flags"]) # the copyable line says the same+ p = self.plan("--model", "q2_0", "--pool-cores", "2-14:2")+ self.assertEqual(p["settings"]["extra"], {"--pool-cores": "2-14:2"})+ p = self.plan("--model", "q2_0") # off by default: nothing is added+ self.assertNotIn("--pool-cores", p["settings"]["extra"])+ p = self.plan("--model", "q2_0", "--pool-cores", "two,four") # a list that cannot be read: a warning, never a refusal+ self.assertNotIn("--pool-cores", p["settings"]["extra"])+ self.assertTrue(any("pool-cores" in w for w in p["warnings"]))+def test_api_key_flags(self):p = self.plan("--model", "q2_0", "--access", "lan", "--api-key", "sk-mine")self.assertEqual(p["settings"]["api_key"], "set")