Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

serve, launch: "pool_cores" in the run config and strata launch --pool-cores

Niko12212aa1ec2d56cd
diff --git a/docs/TERMINAL_LAUNCH.md b/docs/TERMINAL_LAUNCH.md
index 23444e1d1..871bcffb9 100644
--- a/docs/TERMINAL_LAUNCH.md
+++ b/docs/TERMINAL_LAUNCH.md
@@ -62,6 +62,7 @@ strata launch --model iq3_xxs --context 128k --parallel 4 --vision off --tune qu
--access lan --port 8080 --save-preset "Office" --no-tui # serve and print the address
strata launch --model-path /data/models/IQ3_XXS --no-tui # files you already have
strata launch --preset Office --set prefill=65536 --set STRATA_PIN_BY_NEED=1 --no-tui
+strata launch --model q2_0 --pool-cores 2-14:2 --no-tui # experimental: pin the CPU workers to these logical processors (see DETAILS.md)
strata launch --plan --model q2_0 # what would happen; nothing changes
strata launch --model q2_0 --yes --no-tui # no questions (for one: download on a computer with no engine yet)
strata launch --model q2_0 --open # also open the server's page in your browser once it serves
diff --git a/serve/server.py b/serve/server.py
index 1cfadc6af..bf335f9ec 100644
--- a/serve/server.py
+++ b/serve/server.py
@@ -2948,6 +2948,7 @@ def engine_args(cfg: dict) -> list[str]:
if isinstance(seg, int) and not isinstance(seg, bool) and seg > 0 and "--vram-segment-mib" not in args:
args += ["--vram-segment-mib", str(seg)]
args += parallel_args(cfg, args)
+ args += pool_cores_args(cfg, args)
return learned_profile_args(cfg, args)
@@ -2975,6 +2976,32 @@ def parallel_args(cfg: dict, args: list[str]) -> list[str]:
return ["--batch", str(n)]
+_POOL_CORES_OK = re.compile(r"^[0-9]+(-[0-9]+(:[0-9]+)?)?(,[0-9]+(-[0-9]+(:[0-9]+)?)?)*$")
+
+
+def pool_cores_args(cfg: dict, args: list[str]) -> list[str]:
+ """Experimental, off by default: "pool_cores": "0,2,4,6" (or "2-14:2", or a list of numbers) in the config pins the CPU
+ expert workers to exactly those logical processors (--pool-cores; docs/DETAILS.md, "Which logical processors the CPU
+ workers use"). The engine warns about two entries on one core or the host's own and leaves out processors the process may
+ not use; a text this cannot read is said here and ignored (recommend, never force). An engine option already in the args wins."""
+ value = cfg.get("pool_cores")
+ if value is None or value is False or value == "" or value == []:
+ return []
+ if "--pool-cores" in args:
+ return []
+ if isinstance(value, list) and value and all(isinstance(v, int) and not isinstance(v, bool) and v >= 0 for v in value):
+ text = ",".join(str(v) for v in value)
+ elif isinstance(value, str):
+ text = value.replace(" ", "")
+ else:
+ text = ""
+ if not _POOL_CORES_OK.match(text):
+ print(f'[strata] "pool_cores" must be logical processor numbers like "0,2,4,6" or "2-14:2", not {value!r}: ignored '
+ "(the CPU workers keep their default placement)", flush=True)
+ return []
+ return ["--pool-cores", text]
+
+
def profile_shape(path: str) -> tuple[int, int] | None:
"""An expert profile's (layers, experts per layer), from its header (tools/make_profile.py's format), or None
when the file is missing, is not one or is shorter than the pairs its header promises."""
diff --git a/serve/test_pool_cores.py b/serve/test_pool_cores.py
new file mode 100644
index 000000000..ca0c94ca9
--- /dev/null
+++ b/serve/test_pool_cores.py
@@ -0,0 +1,41 @@
+"""The run config's experimental "pool_cores" (the CPU expert workers' logical processors; --pool-cores in the engine)."""
+import contextlib
+import io
+import unittest
+
+from serve.server import engine_args, pool_cores_args
+
+
+class PoolCoresConfig(unittest.TestCase):
+ def test_off_by_default(self):
+ self.assertEqual(pool_cores_args({}, []), [])
+ self.assertEqual(pool_cores_args({"pool_cores": None}, []), [])
+ self.assertEqual(pool_cores_args({"pool_cores": ""}, []), [])
+ self.assertEqual(pool_cores_args({"pool_cores": []}, []), [])
+ self.assertEqual(pool_cores_args({"pool_cores": False}, []), [])
+ self.assertEqual(engine_args({"args": ["--pack", "p"]}), ["--pack", "p"]) # nothing is added to an ordinary config
+
+ def test_text_and_list_forms(self):
+ self.assertEqual(pool_cores_args({"pool_cores": "2,4,6,8"}, []), ["--pool-cores", "2,4,6,8"])
+ self.assertEqual(pool_cores_args({"pool_cores": " 2-14:2 "}, []), ["--pool-cores", "2-14:2"])
+ self.assertEqual(pool_cores_args({"pool_cores": [2, 4, 6]}, []), ["--pool-cores", "2,4,6"])
+ self.assertEqual(pool_cores_args({"pool_cores": "0,2-6,10"}, []), ["--pool-cores", "0,2-6,10"])
+
+ def test_the_args_own_value_wins(self):
+ self.assertEqual(pool_cores_args({"pool_cores": "2,4"}, ["--pool-cores", "6,8"]), [])
+
+ def test_a_list_that_cannot_be_read_is_said_and_ignored(self):
+ for bad in ("two,four", "1,,2", "6-2x", -1, True, 5, [1, "a"], [-1], {"a": 1}):
+ buf = io.StringIO()
+ with contextlib.redirect_stdout(buf):
+ r = pool_cores_args({"pool_cores": bad}, [])
+ self.assertEqual(r, [], bad)
+ self.assertIn("pool_cores", buf.getvalue(), bad)
+
+ def test_reaches_the_engine_arguments(self):
+ args = engine_args({"args": ["--pack", "p"], "pool_cores": "2,4,6"})
+ self.assertEqual(args[-2:], ["--pool-cores", "2,4,6"])
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/strata_cli/launch/cmd.py b/strata_cli/launch/cmd.py
index bd401cd3f..c293b6b28 100644
--- a/strata_cli/launch/cmd.py
+++ b/strata_cli/launch/cmd.py
@@ -5,6 +5,7 @@ from __future__ import annotations
import argparse
import json
import os
+import re
import sys
import threading
from pathlib import Path
@@ -103,6 +104,10 @@ def add_parsers(sp: argparse._SubParsersAction) -> list[argparse.ArgumentParser]
g.add_argument("--tune-allow-small", action="store_true", help="let the tuner also try settings that can change answers very slightly")
g.add_argument("--set", action="append", default=[], metavar="KEY=VALUE",
help="an advanced engine setting or STRATA_* variable, e.g. prefill=65536 (repeat); unknown names get a warning, not a refusal")
+ g.add_argument("--pool-cores", metavar="LIST",
+ help="experimental: pin the CPU workers to exactly these logical processors, e.g. 2,4,6,8 or 2-14:2 (the numbers Task Manager or "
+ "lscpu show). The default is one worker per physical core, which the engine's start line names; "
+ "two entries on one core are kept with a warning")
a = p.add_argument_group("who can use it")
a.add_argument("--access", type=_choice("a way to share Strata", S.ACCESS_KINDS), metavar="local|lan|cloudflare|tailscale",
help="local: this computer only; lan: your network; cloudflare: a quick tunnel; tailscale: your tailnet")
@@ -235,6 +240,13 @@ def settings_from_args(a: argparse.Namespace, catalog=None) -> tuple[S.LaunchSet
s.no_key = False
if a.set:
s.extra = {**s.extra, **S.settings_from_pairs(a.set)}
+ if a.pool_cores:
+ text = a.pool_cores.replace(" ", "")
+ if not re.fullmatch(r"\d+(-\d+(:\d+)?)?(,\d+(-\d+(:\d+)?)?)*", text):
+ notices.append(S.Notice("warn", f"'{a.pool_cores}' is not a list of logical processor numbers, so --pool-cores is left out.",
+ "Write it like 2,4,6,8 or 2-14:2. The CPU workers keep their default placement."))
+ else:
+ s.extra = {**s.extra, "--pool-cores": text}
if a.save_preset:
s.name = a.save_preset
return s, notices
@@ -587,6 +599,7 @@ def run_launch(a: argparse.Namespace) -> int:
except LaunchError as e:
return fail(f"{e.text}" + (f" {e.next}" if e.next else ""), e.code)
s, notes = apply_defaults(s, catalog)
+ notes = [*notices, *notes] # what the flags themselves said (a --pool-cores list that could not be read, a key beside --no-api-key)
errors = [n for n in notes if n.level == "error"]
if errors:
return fail(errors[0].line(), 2)
diff --git a/tests/terminal/test_launch.py b/tests/terminal/test_launch.py
index e471c9612..13ad5837c 100644
--- a/tests/terminal/test_launch.py
+++ b/tests/terminal/test_launch.py
@@ -794,6 +794,20 @@ class FlagsCommand(unittest.TestCase):
self.assertEqual(s["extra"], {"--prefill": "65536", "STRATA_PIN_BY_NEED": "1"})
self.assertIn("--set prefill=65536", p["flags"])
+ def test_pool_cores_is_an_experimental_advanced_setting(self):
+ code, out, err = run_cli(["launch", "--help"])
+ self.assertIn("--pool-cores", out)
+ p = self.plan("--model", "q2_0", "--pool-cores", "2,4, 6,8")
+ self.assertEqual(p["settings"]["extra"], {"--pool-cores": "2,4,6,8"})
+ self.assertIn("--set pool-cores=2,4,6,8", p["flags"]) # the copyable line says the same
+ p = self.plan("--model", "q2_0", "--pool-cores", "2-14:2")
+ self.assertEqual(p["settings"]["extra"], {"--pool-cores": "2-14:2"})
+ p = self.plan("--model", "q2_0") # off by default: nothing is added
+ self.assertNotIn("--pool-cores", p["settings"]["extra"])
+ p = self.plan("--model", "q2_0", "--pool-cores", "two,four") # a list that cannot be read: a warning, never a refusal
+ self.assertNotIn("--pool-cores", p["settings"]["extra"])
+ self.assertTrue(any("pool-cores" in w for w in p["warnings"]))
+
def test_api_key_flags(self):
p = self.plan("--model", "q2_0", "--access", "lan", "--api-key", "sk-mine")
self.assertEqual(p["settings"]["api_key"], "set")