Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

Split serving, first version: GEN 0 prefill=1 in the engine, serve/split.py (SplitEngine, config split_serve / --split-serve), tests

Niko1221690172c464ef
diff --git a/serve/server.py b/serve/server.py
index 76b5a2c79..8fa45c4d4 100644
--- a/serve/server.py
+++ b/serve/server.py
@@ -2183,7 +2183,7 @@ class StrataEngine:
"tok_s": round(r["generated"] / max(1e-6, now - ft), 1) if ft else None})
return view
- def generate(self, ids, max_new, sampling, cancel, embeddings=None):
+ def generate(self, ids, max_new, sampling, cancel, embeddings=None, prefill_only=False):
"""Yields token ids, and None as a heartbeat every 10 s while the engine is quiet (reading a long prompt):
the HTTP layer turns it into an SSE comment, which keeps clients' watchdogs calm and notices a client that
has gone. A consumer that stops early (or `cancel`) makes the engine STOP, so it does not run to max_new."""
@@ -2198,6 +2198,10 @@ class StrataEngine:
# an image request takes the same sampling keys as text (#75: it used to decode greedily whatever was asked)
head = f"GENI {int(max_new)}{self.sampling_keys(sampling or {})} {embeddings}" if embeddings else \
f"GEN {int(max_new)}{self.sampling_keys(sampling or {})}"
+ if prefill_only: # split serving (serve/split.py): read the prompt, make no token
+ if embeddings or int(max_new) != 0:
+ raise ValueError("prefill_only is a text prompt and max_new 0")
+ head = "GEN 0 prefill=1"
try:
self.proc.stdin.write(f"{head} {','.join(str(int(t)) for t in ids)}\n")
self.proc.stdin.flush()
@@ -4639,6 +4643,8 @@ class Service:
live["state"] = "generating"
if hasattr(self.engine, "replica_view"): # opt-in replicas: each one's state and requests routed to it
live["replicas"] = self.engine.replica_view()
+ if hasattr(self.engine, "split_view"): # opt-in split serving: the hand-overs so far (serve/split.py)
+ live["split_serve"] = self.engine.split_view()
engine = {"model": self.model, "max_context": self.reported_ctx(), "images": self.vision is not None,
**dict(getattr(self.engine, "info", {}) or {})}
tel = self.telemetry.snapshot() if getattr(self, "telemetry", None) else {"now": {}, "history": {}, "static": {}}
@@ -8008,6 +8014,39 @@ def start_replicas(cfg: dict, groups: list[list[int]], exe: str, effort_end: lis
return rp.ReplicaEngine(engines, load_tokens=load)
+def start_split(cfg: dict, spec: dict, exe: str, effort_end: list[str], lazy: bool, *, offline: bool = False):
+ """Opt-in split serving (serve/split.py): a prefill engine and a decode engine, one card each, the same model and the
+ same session settings. Started one after the other, so a shared expert arena is filled once; the SplitEngine in
+ front hands a long prompt from the first to the second."""
+ from serve import split as sp
+ sp.check_cfg(cfg, spec)
+ arena, arena_note = sp.arena_path(cfg, spec)
+ work = sp.session_dir(spec)
+ print(f"[strata] split serving: the prompt is read on card {spec['prefill_gpu']}, the answer is made on card "
+ f"{spec['decode_gpu']} (prompts with {spec['min_prompt_tokens']}+ new tokens are handed over); {arena_note}",
+ flush=True)
+ engines = {}
+ try:
+ for role in ("prefill", "decode"):
+ rc = sp.role_cfg(cfg, spec, role)
+ if arena:
+ rc["args"] = list(rc["args"]) + ["--shared-expert-arena", arena]
+ if role == "prefill":
+ warn_budget_over_ram(engine_args(rc) if "args" in rc else [])
+ engines[role] = StrataEngine(exe, engine_args(rc) + effort_end, cwd=rc.get("cwd"), log=rc.get("log"),
+ env=child_env(rc, offline=offline), lazy=lazy)
+ except BaseException:
+ for e in engines.values():
+ try:
+ e.close()
+ except Exception: # noqa: BLE001
+ pass
+ shutil.rmtree(work, ignore_errors=True)
+ raise
+ return sp.SplitEngine(engines["prefill"], engines["decode"], min_tokens=spec["min_prompt_tokens"], work_dir=work,
+ arena_file=arena)
+
+
def check_request_sampling(req) -> None:
"""A request's sampling field of the wrong JSON type ("temperature": "0.7", "top_k": 40.0, "seed": true) is a 400
that names the field, as the same value is through POST /settings or the run config. Unchecked, the value
@@ -8211,6 +8250,10 @@ def main(argv=None, *, lifecycle_event=None, interrupt_exit_code: int = 0) -> in
help="data-parallel replicas (opt-in): cut the \"gpu\" list into N equal groups and run one engine on "
"each, requests spread over them (also \"replicas\" in the config: a number, or "
"[{\"gpus\": [0, 1]}, {\"gpus\": [2, 3]}])")
+ ap.add_argument("--split-serve", default=None, metavar="PREFILL,DECODE",
+ help="split serving (opt-in): one engine on card PREFILL reads the prompt, a second on card DECODE makes "
+ "the answer, the session handed over in RAM (also \"split_serve\" in the config: "
+ "{\"prefill_gpu\": 0, \"decode_gpu\": 1}; docs/SPLIT_SERVE.md)")
launch_args = list(sys.argv[1:] if argv is None else argv)
a = ap.parse_args(launch_args)
if a.engine == "strata" and (_rt := rttime_limit_note()): # #3238: say it before the first unexplained -9
@@ -8284,6 +8327,12 @@ def main(argv=None, *, lifecycle_event=None, interrupt_exit_code: int = 0) -> in
lift_reasoning_effort_arg(cfg)
if a.replicas is not None:
cfg["replicas"] = a.replicas
+ if a.split_serve is not None:
+ pair = [x.strip() for x in a.split_serve.split(",")]
+ if len(pair) != 2 or not all(x.isdecimal() for x in pair):
+ ap.error("--split-serve takes two card numbers: PREFILL,DECODE (for example 0,1)")
+ cfg["split_serve"] = {**(cfg.get("split_serve") if isinstance(cfg.get("split_serve"), dict) else {}),
+ "prefill_gpu": int(pair[0]), "decode_gpu": int(pair[1])}
if a.gpu is not None:
cfg["gpu"] = int(a.gpu) if a.gpu.strip().isdigit() else a.gpu
a.host = a.host or cfg.get("host") or "127.0.0.1" # issue #26: the run scripts pass no --host, the config can
@@ -8412,15 +8461,24 @@ def main(argv=None, *, lifecycle_event=None, interrupt_exit_code: int = 0) -> in
groups = rp.gpu_groups(cfg)
except rp.ReplicaConfigError as e:
raise SystemExit(f"[strata] config {e}")
+ from serve import split as split_mod
+ try:
+ split = split_mod.split_spec(cfg)
+ if split:
+ split_mod.check_cfg(cfg, split)
+ except split_mod.SplitConfigError as e:
+ raise SystemExit(f"[strata] config {e}")
if groups:
engine = start_replicas(cfg, groups, exe, effort_end or [], lazy, offline=a.offline)
+ elif split:
+ engine = start_split(cfg, split, exe, effort_end or [], lazy, offline=a.offline)
else:
engine = StrataEngine(exe, engine_args(cfg) + (effort_end or []), cwd=cfg.get("cwd"), log=cfg.get("log"),
env=env, lazy=lazy)
engine.silence_s = silence # an attribute of its own: restart() keeps it
warn_tight_ram(engine.info.get("arena_mib"))
note = desktop_vram_note(cfg.get("backend"), engine.info.get("vram_free_mib"), engine.spawn[1],
- linux_desktop()) if not groups else None
+ linux_desktop()) if not groups and not split else None
if note: # #560 #516: before --open starts a browser on that card
print(note, flush=True)
else:
@@ -8588,8 +8646,12 @@ def main(argv=None, *, lifecycle_event=None, interrupt_exit_code: int = 0) -> in
svc.gpu_indices = [c for g in groups for c in g]
svc.gpu_index = svc.gpu_indices[0]
svc.replica_gpu_groups = groups
+ if a.engine == "strata" and split: # split serving: the two cards (the prefill card first)
+ svc.gpu_indices = [split["prefill_gpu"], split["decode_gpu"]]
+ svc.gpu_index = split["decode_gpu"]
svc.terminal_visibility_order = visible_device_order(env, cfg.get("backend")) if a.engine == "strata" else []
svc.terminal_topology_mode = ("replicas (configured)" if a.engine == "strata" and groups
+ else "split serve (configured)" if a.engine == "strata" and split
else "layer split (configured)" if a.engine == "strata" and layer_split_of(cfg)
else "single GPU or CPU" if a.engine == "strata" else "mock")
svc.terminal_configured_split = (layer_split_value(cfg) if a.engine == "strata" and layer_split_of(cfg)
diff --git a/serve/split.py b/serve/split.py
new file mode 100644
index 000000000..5b28394a1
--- /dev/null
+++ b/serve/split.py
@@ -0,0 +1,488 @@
+"""Split serving: one engine reads the prompt, another makes the answer (issue #4000, idea and manual setup by @MarcoTool).
+
+Two engines on the same PC, one card each, the same model files and the same session settings. A long prompt is read by
+the prefill engine, which saves the session (the engine's own session file: SAVE <path>) into RAM; the decode engine
+restores it (RESTORE <path>) and makes the answer. The server keeps its one HTTP front, its one Service and its one set of
+endpoints; `SplitEngine` stands in for the engine, as `ReplicaEngine` does for replicas.
+
+Opt-in, in the run config (nothing changes without it):
+
+ "split_serve": {"prefill_gpu": 0, "decode_gpu": 1}
+
+ prefill_gpu / decode_gpu the card of each engine, numbered as nvidia-smi numbers them (required, different)
+ min_prompt_tokens hand a request over only when the decode engine would read at least this many new tokens
+ itself (default 2048); a shorter tail is read by the decode engine in place
+ dir where the session files go (default /dev/shm when it has room, else the temp folder)
+ shared_arena true (default): one copy of the experts in RAM for both engines (Linux, a file in /dev/shm
+ when the experts fit there); false: each engine keeps its own copy; a path: that file
+ prefill_args / decode_args extra engine arguments for one role (a bigger expert cache for the decode engine, say)
+
+What must match between the two engines is what the session file checks (docs/SPLIT_SERVE.md): the model files, the engine
+version, the backend, --kv, --max-context, --kv-resident, --mtp-window, the rope settings and the arithmetic switches. They
+start from one config, so they do. Every failure on the way (a prefill engine that died, a refused file) is not an error
+for the client: the decode engine reads the prompt itself, as it would without the option.
+
+Honest limit of this first version: one request at a time (the Service's FIFO), one conversation resident on the decode
+engine (a restore replaces what it held), no parallel slots (--batch), no layer split in an engine, no pictures (a request
+with pictures goes to the decode engine alone).
+"""
+from __future__ import annotations
+
+import os
+import shutil
+import tempfile
+import threading
+import time
+
+MIN_TOKENS = 2048 # the decode engine's own tail below this is read in place
+KNOWN_KEYS = {"prefill_gpu", "decode_gpu", "min_prompt_tokens", "dir", "shared_arena", "prefill_args", "decode_args"}
+FORBIDDEN_ARGS = { # engine arguments that cannot be used in a split-serve engine, and why
+ "--layer-split": "an engine of the split runs on one card",
+ "--split-device": "an engine of the split runs on one card",
+ "--peer-device": "session files do not support --peer-device",
+ "--batch": "session files do not support --batch (parallel requests)",
+ "--slots": "session files do not support parallel slots",
+}
+HEAL_COOLDOWN_S = 60.0 # a prefill engine that died is started again at most this often
+HEARTBEAT_S = 5.0 # a long file step sends a keep-alive this often
+
+
+class SplitConfigError(ValueError):
+ pass
+
+
+def split_spec(cfg: dict) -> dict | None:
+ """The checked "split_serve" section of the run config, or None when the option is off."""
+ raw = cfg.get("split_serve")
+ if raw is None or raw is False:
+ return None
+ if not isinstance(raw, dict):
+ raise SplitConfigError('"split_serve" must be {"prefill_gpu": N, "decode_gpu": M}, not ' + repr(raw))
+ unknown = sorted(set(raw) - KNOWN_KEYS)
+ if unknown:
+ raise SplitConfigError(f'"split_serve" has no key {unknown[0]!r} (keys: {", ".join(sorted(KNOWN_KEYS))})')
+ cards = []
+ for k in ("prefill_gpu", "decode_gpu"):
+ v = raw.get(k)
+ if isinstance(v, str) and v.strip().isdigit():
+ v = int(v.strip())
+ if isinstance(v, bool) or not isinstance(v, int) or v < 0:
+ raise SplitConfigError(f'"split_serve" needs "{k}": the number of a card (0, 1, ...), not {v!r}')
+ cards.append(v)
+ if cards[0] == cards[1]:
+ raise SplitConfigError(f'"split_serve": prefill_gpu and decode_gpu are both card {cards[0]}; each engine needs a '
+ "card of its own")
+ out = {"prefill_gpu": cards[0], "decode_gpu": cards[1]}
+ n = raw.get("min_prompt_tokens", MIN_TOKENS)
+ if isinstance(n, bool) or not isinstance(n, int) or n < 0:
+ raise SplitConfigError(f'"split_serve": "min_prompt_tokens" must be a number of tokens (0 or more), not {n!r}')
+ out["min_prompt_tokens"] = n
+ d = raw.get("dir")
+ if d is not None and (not isinstance(d, str) or not d.strip()):
+ raise SplitConfigError(f'"split_serve": "dir" must be a folder name, not {d!r}')
+ out["dir"] = d
+ sa = raw.get("shared_arena", True)
+ if not (isinstance(sa, bool) or (isinstance(sa, str) and sa.strip())):
+ raise SplitConfigError(f'"split_serve": "shared_arena" must be true, false or a file name, not {sa!r}')
+ out["shared_arena"] = sa
+ for k in ("prefill_args", "decode_args"):
+ a = raw.get(k, [])
+ if not isinstance(a, list) or not all(isinstance(x, (str, int, float)) and not isinstance(x, bool) for x in a):
+ raise SplitConfigError(f'"split_serve": "{k}" must be a list of engine arguments')
+ out[k] = [str(x) for x in a]
+ return out
+
+
+def check_cfg(cfg: dict, spec: dict) -> None:
+ """Refuse (with the reason, in plain words) a config the split cannot run."""
+ if cfg.get("replicas") not in (None, False):
+ raise SplitConfigError('"split_serve" and "replicas" cannot be used together: both start engines on the cards')
+ args = [str(a) for a in cfg.get("args") or []] + spec["prefill_args"] + spec["decode_args"]
+ for flag, why in FORBIDDEN_ARGS.items():
+ if flag in args:
+ raise SplitConfigError(f'"split_serve" cannot be used with {flag} in the engine arguments: {why}')
+ for i, a in enumerate(args):
+ if a == "--prompt-cache" and i + 1 < len(args) and args[i + 1].strip() == "0":
+ raise SplitConfigError('"split_serve" needs the conversation cache (--prompt-cache 1 or more): the session '
+ "file is made from it")
+ if cfg.get("batch") or cfg.get("parallel"):
+ raise SplitConfigError('"split_serve" does not run parallel requests ("parallel"/"batch"): remove it')
+
+
+def _suffixed(path, tag: str):
+ if not isinstance(path, str) or not path:
+ return path
+ root, ext = os.path.splitext(path)
+ return f"{root}.{tag}{ext}"
+
+
+def role_cfg(cfg: dict, spec: dict, role: str) -> dict:
+ """The config of one engine of the split: its own card, log and learned-profile file; its role's extra arguments."""
+ import copy
+ c = copy.deepcopy(cfg)
+ c.pop("split_serve", None)
+ c["gpu"] = spec[f"{role}_gpu"]
+ c.pop("hip_ordinal", None)
+ c.pop("gpu_roles", None) # the roles were for a layer split over several cards
+ c["log"] = _suffixed(cfg.get("log"), role)
+ if cfg.get("expert_profile_save"):
+ c["expert_profile_save"] = _suffixed(cfg["expert_profile_save"], role)
+ extra = spec[f"{role}_args"]
+ if extra:
+ c["args"] = list(c["args"]) + extra
+ return c
+
+
+def _arg_value(args: list, flag: str):
+ for i, a in enumerate(args):
+ if a == flag and i + 1 < len(args):
+ return args[i + 1]
+ return None
+
+
+def arena_path(cfg: dict, spec: dict) -> tuple[str | None, str]:
+ """(file for --shared-expert-arena or None, a line for the log). Only Linux can share the arena; the experts
+ file must fit /dev/shm (a tmpfs is half of the RAM by default) unless the config names another file."""
+ args = [str(a) for a in cfg.get("args") or []]
+ if "--shared-expert-arena" in args:
+ return None, "the config already names a shared expert arena"
+ sa = spec["shared_arena"]
+ if sa is False:
+ return None, "shared_arena is off: each engine keeps its own copy of the experts in RAM"
+ if os.name == "nt":
+ return None, ("this OS cannot share the expert arena between two engines (Linux only): each engine keeps its own "
+ "copy of the experts in RAM, so the PC needs RAM for two")
+ if isinstance(sa, str):
+ return os.path.abspath(sa), f"shared expert arena in {os.path.abspath(sa)}"
+ pack = _arg_value(args, "--pack")
+ experts = os.path.join(pack, "experts.bin") if pack else None
+ if not experts or not os.path.isfile(experts):
+ return None, ("no experts.bin to size a shared arena from (a pack that reads the GGUF in place): each engine "
+ "keeps its own copy of the experts; name a file in \"shared_arena\" to share one")
+ need = os.path.getsize(experts)
+ try:
+ free = shutil.disk_usage("/dev/shm").free
+ except OSError:
+ free = 0
+ if free < need + (1 << 30):
+ return None, (f"/dev/shm has {free / 2**30:.1f} GiB free and the experts are {need / 2**30:.1f} GiB: each engine "
+ "keeps its own copy of the experts in RAM (two copies). To share one copy name a file in "
+ "\"shared_arena\" (a file on disk works, it is written once) or make /dev/shm bigger")
+ return "/dev/shm/strata-split-arena.bin", f"one shared expert arena in /dev/shm ({need / 2**30:.1f} GiB)"
+
+
+def session_dir(spec: dict) -> str:
+ d = spec.get("dir")
+ if d:
+ os.makedirs(d, exist_ok=True)
+ return d
+ shm = "/dev/shm"
+ try:
+ if os.name != "nt" and os.path.isdir(shm) and os.access(shm, os.W_OK) and shutil.disk_usage(shm).free > (6 << 30):
+ return tempfile.mkdtemp(prefix="strata-split-", dir=shm)
+ except OSError:
+ pass
+ return tempfile.mkdtemp(prefix="strata-split-")
+
+
+def common_prefix(a, b) -> int:
+ n = min(len(a), len(b))
+ i = 0
+ while i < n and a[i] == b[i]:
+ i += 1
+ return i
+
+
+class SplitEngine:
+ """Stands in for one engine (the interface Service uses), over a prefill engine and a decode engine."""
+
+ NOT_HERE = {"session_file", "vram", "ctl", "slot_cv"} # not offered: one conversation file / VRAM resize / control line
+ PROPAGATE = {"silence_s", "gpu_busy"}
+
+ def __init__(self, prefill, decode, *, min_tokens: int = MIN_TOKENS, work_dir: str | None = None,
+ arena_file: str | None = None, say=print):
+ d = self.__dict__
+ d["prefill"], d["decode"] = prefill, decode
+ d["cur"] = decode # the engine the request is on now (progress, last)
+ d["min_tokens"] = int(min_tokens)
+ d["work_dir"] = work_dir or tempfile.mkdtemp(prefix="strata-split-")
+ d["arena_file"] = arena_file
+ d["say"] = say
+ d["held"] = [] # the tokens the decode engine's session holds (an estimate: ids + answer)
+ d["seq"] = 0
+ d["heal_at"] = 0.0
+ d["healing"] = False
+ d["lock"] = threading.Lock()
+ d["stats"] = {"requests": 0, "handed_over": 0, "fallbacks": 0, "short": 0, "bytes": 0, "prefill_ms": 0.0,
+ "save_ms": 0.0, "restore_ms": 0.0, "last": None}
+
+ # ---- attribute plumbing
+ def __getattr__(self, name):
+ if name in self.NOT_HERE or name.startswith("__"):
+ raise AttributeError(name)
+ return getattr(self.decode, name)
+
+ def __setattr__(self, name, value):
+ if name in self.PROPAGATE:
+ self.prefill.__setattr__(name, value)
+ self.decode.__setattr__(name, value)
+ else:
+ self.__dict__[name] = value
+
+ @property
+ def last(self):
+ return self.decode.last
+
+ @last.setter
+ def last(self, value):
+ self.decode.last = value
+
+ @property
+ def progress(self):
+ return self.cur.progress
+
+ @property
+ def prefill_tok_s_mean(self):
+ return self.cur.prefill_tok_s_mean
+
+ @property
+ def batch(self) -> int:
+ return 0
+
+ @property
+ def max_context(self) -> int:
+ v = [e.max_context for e in (self.prefill, self.decode) if getattr(e, "max_context", 0) and e.alive()]
+ return min(v) if v else 0
+
+ @property
+ def unloaded(self) -> bool:
+ return all(getattr(e, "unloaded", False) for e in (self.prefill, self.decode))
+
+ @property
+ def starting(self) -> bool:
+ return not self.alive() and any(getattr(e, "starting", False) for e in (self.prefill, self.decode))
+
+ @property
+ def info(self) -> dict:
+ i = dict(self.decode.info)
+ i["split_serve"] = True
+ i["split_prefill_alive"] = bool(self.prefill.alive())
+ return i
+
+ @info.setter
+ def info(self, value):
+ pass
+
+ def alive(self) -> bool:
+ return bool(self.decode.alive())
+
+ def exit_code(self):
+ return next((c for c in (e.exit_code() for e in (self.decode, self.prefill) if not e.alive()) if c is not None),
+ None)
+
+ def death_note(self) -> str:
+ return next((e.death_note() for e in (self.decode, self.prefill) if not e.alive()), "")
+
+ def split_view(self) -> dict:
+ """For /metrics: how the two engines are used."""
+ s = dict(self.stats)
+ s.update(prefill_alive=bool(self.prefill.alive()), decode_alive=bool(self.decode.alive()),
+ min_prompt_tokens=self.min_tokens)
+ return s
+
+ # ---- life cycle
+ def restart(self, tries: int = 3):
+ errors = {}
+ for name, e in (("decode", self.decode), ("prefill", self.prefill)): # one at a time: the arena fills once
+ if e.alive():
+ continue
+ try:
+ e.restart(tries)
+ except BaseException as x: # noqa: BLE001
+ errors[name] = x
+ if "decode" in errors:
+ raise errors["decode"]
+ for name, x in errors.items():
+ self.say(f"[strata] the {name} engine of the split did not start ({x}); the other one serves")
+ self.__dict__["held"] = []
+
+ def unload(self):
+ for e in (self.prefill, self.decode):
+ e.unload()
+ self.__dict__["held"] = []
+
+ def close(self):
+ err = None
+ for e in (self.prefill, self.decode):
+ try:
+ e.close()
+ except BaseException as x: # noqa: BLE001
+ err = err or x
+ shutil.rmtree(self.work_dir, ignore_errors=True)
+ if self.arena_file and self.arena_file.startswith("/dev/shm/strata-split-"):
+ try:
+ os.unlink(self.arena_file) # RAM until it is gone
+ except OSError:
+ pass
+ if err:
+ raise err
+
+ def heal(self) -> None:
+ """Start a prefill engine that died again, in the background (the decode engine serves meanwhile)."""
+ e = self.prefill
+ now = time.monotonic()
+ if e.alive() or getattr(e, "unloaded", False) or getattr(e, "starting", False) or self.healing or now < self.heal_at:
+ return
+ self.__dict__["heal_at"] = now + HEAL_COOLDOWN_S
+ self.__dict__["healing"] = True
+
+ def run():
+ try:
+ self.say("[strata] split: the prefill engine had stopped; starting it again")
+ e.restart()
+ self.say("[strata] split: the prefill engine is running again")
+ except BaseException as x: # noqa: BLE001
+ self.say(f"[strata] split: the prefill engine did not start again: {x}")
+ finally:
+ self.__dict__["healing"] = False
+ threading.Thread(target=run, daemon=True).start()
+
+ # ---- the request
+ def wants_handover(self, ids) -> tuple[bool, int]:
+ """(hand the prompt to the prefill engine?, the tokens the decode engine would read itself)."""
+ n = len(ids)
+ if n < 2:
+ return False, 0
+ need = (n - 1) - min(common_prefix(self.held, ids), n - 1)
+ return need >= max(1, self.min_tokens) and self.prefill.alive(), need
+
+ def _wait(self, fn):
+ """Run a blocking engine call on a thread; yields None as a keep-alive while it runs. -> its result."""
+ box = {}
+
+ def run():
+ try:
+ box["r"] = fn()
+ except BaseException as e: # noqa: BLE001
+ box["e"] = e
+ th = threading.Thread(target=run, daemon=True)
+ th.start()
+ while th.is_alive():
+ th.join(HEARTBEAT_S)
+ if th.is_alive():
+ yield None
+ if "e" in box:
+ raise box["e"]
+ return box.get("r")
+
+ def _hand_over(self, ids, cancel, rec: dict):
+ """Prefill engine reads the prompt, saves the session; the decode engine restores it. Yields keep-alives.
+ rec['ok'] is True when the decode engine now holds the prompt's first n-1 tokens."""
+ from serve.server import EngineDied, SessionRefused
+ P, D = self.prefill, self.decode
+ n = len(ids)
+ self.__dict__["seq"] += 1
+ path = os.path.join(self.work_dir, f"handover-{os.getpid()}-{self.seq}.session")
+ rec.update(ok=False)
+ t0 = time.monotonic()
+ self.__dict__["cur"] = P
+ try:
+ try:
+ for _ in P.generate(ids, 0, {}, cancel, prefill_only=True):
+ yield None
+ except (EngineDied, ValueError) as e:
+ self.say(f"[strata] split: the prefill engine failed ({e}); the decode engine reads the prompt itself")
+ self.heal()
+ return
+ if cancel.is_set():
+ return
+ p_last = dict(P.last or {})
+ rec["prefill_ms"] = (time.monotonic() - t0) * 1000.0
+ rec["p_last"] = p_last
+ try:
+ saved = yield from self._wait(lambda: P.session_file("save", path))
+ self.__dict__["cur"] = D
+ restored = yield from self._wait(lambda: D.session_file("restore", path))
+ except SessionRefused as e:
+ self.say(f"[strata] split: the session file was refused ({e}); the decode engine reads the prompt itself")
+ return
+ except EngineDied as e:
+ if not D.alive():
+ raise
+ self.say(f"[strata] split: the prefill engine failed while saving ({e}); the decode engine reads the prompt "
+ "itself")
+ self.heal()
+ return
+ rec.update(save_ms=float(saved["ms"]), restore_ms=float(restored["ms"]), bytes=int(saved["bytes"]),
+ tokens=int(restored["tokens"]), ok=restored["tokens"] == n - 1)
+ if not rec["ok"]:
+ self.say(f"[strata] split: the decode engine restored {restored['tokens']} tokens, not the {n - 1} the "
+ "prompt needs; it reads the rest itself")
+ rec["ok"] = restored["tokens"] > 0
+ finally:
+ self.__dict__["cur"] = D
+ try:
+ os.unlink(path)
+ except OSError:
+ pass
+
+ def generate(self, ids, max_new, sampling, cancel, embeddings=None):
+ D = self.decode
+ ids = list(ids)
+ st = self.stats
+ st["requests"] += 1
+ rec: dict = {}
+ use, need = (False, 0) if embeddings is not None else self.wants_handover(ids)
+ if not use:
+ st["short"] += 1
+ if use:
+ if not self.prefill.alive():
+ self.heal()
+ yield from self._hand_over(ids, cancel, rec)
+ if cancel.is_set():
+ self.__dict__["held"] = []
+ return
+ if rec.get("ok"):
+ st["handed_over"] += 1
+ st["bytes"] += rec["bytes"]
+ st["prefill_ms"] += rec["prefill_ms"]
+ st["save_ms"] += rec["save_ms"]
+ st["restore_ms"] += rec["restore_ms"]
+ self.__dict__["held"] = ids[:len(ids) - 1]
+ else:
+ st["fallbacks"] += 1
+ self.__dict__["cur"] = D
+ produced: list[int] = []
+ ok = False
+ try:
+ gen = D.generate(ids, max_new, sampling, cancel, embeddings=embeddings) if embeddings is not None \
+ else D.generate(ids, max_new, sampling, cancel)
+ for t in gen:
+ if t is not None:
+ produced.append(t)
+ yield t
+ ok = True
+ finally:
+ self.__dict__["held"] = (ids + produced) if ok else []
+ if rec.get("ok") and ok:
+ self._account(ids, rec)
+
+ def _account(self, ids, rec: dict) -> None:
+ """The figures of the engine's last DONE, as the whole request: the prefill engine's read, the file steps and the
+ decode engine's own start are the prompt time; what the prefill engine reused is the cached part."""
+ D = self.decode
+ last = dict(D.last or {})
+ p = rec.get("p_last") or {}
+ total_prompt_ms = rec["prefill_ms"] + rec["save_ms"] + rec["restore_ms"] + float(last.get("prompt_ms") or 0.0)
+ p_read = p.get("prompt_read")
+ if p_read is None:
+ p_read = len(ids) - int(p.get("reused") or 0)
+ last.update(prompt_ms=total_prompt_ms, reused=int(p.get("reused") or 0),
+ prompt_read=max(0, min(len(ids), int(p_read) + int(last.get("prompt_read") or 0))),
+ split_prefill_ms=round(rec["prefill_ms"], 1), split_save_ms=round(rec["save_ms"], 1),
+ split_restore_ms=round(rec["restore_ms"], 1), split_bytes=rec["bytes"])
+ D.last = last
+ self.stats["last"] = {k: last[k] for k in last if k.startswith("split_")}
+ self.say(f"[strata] split: prompt {len(ids)} tokens read on the prefill card in {rec['prefill_ms'] / 1000:.1f} s, "
+ f"session {rec['bytes'] / 2**20:.0f} MiB saved in {rec['save_ms'] / 1000:.2f} s and restored in "
+ f"{rec['restore_ms'] / 1000:.2f} s; the decode card makes the answer")
diff --git a/serve/test_split.py b/serve/test_split.py
new file mode 100644
index 000000000..a12164363
--- /dev/null
+++ b/serve/test_split.py
@@ -0,0 +1,271 @@
+"""Split serving (serve/split.py): the config, the decision, the hand-over with two fake engines, and every fallback.
+The engine-side proof (the continuation after a hand-over equals a single engine's) is tests/split_serve_exact.py."""
+import os
+import tempfile
+import threading
+import unittest
+
+from serve import split as sp
+from serve.server import EngineDied, SessionRefused
+from serve.split import SplitConfigError, SplitEngine, check_cfg, common_prefix, role_cfg, split_spec
+
+
+class Config(unittest.TestCase):
+ def test_off_without_the_key(self):
+ self.assertIsNone(split_spec({}))
+ self.assertIsNone(split_spec({"split_serve": False}))
+ self.assertIsNone(split_spec({"split_serve": None}))
+
+ def test_cards_and_defaults(self):
+ s = split_spec({"split_serve": {"prefill_gpu": 0, "decode_gpu": "1"}})
+ self.assertEqual((s["prefill_gpu"], s["decode_gpu"]), (0, 1))
+ self.assertEqual(s["min_prompt_tokens"], sp.MIN_TOKENS)
+ self.assertIs(s["shared_arena"], True)
+ self.assertEqual((s["prefill_args"], s["decode_args"]), ([], []))
+
+ def test_mistakes_are_said(self):
+ bad = [{"split_serve": 1}, {"split_serve": {"prefill_gpu": 0}}, {"split_serve": {"prefill_gpu": 1, "decode_gpu": 1}},
+ {"split_serve": {"prefill_gpu": 0, "decode_gpu": -1}}, {"split_serve": {"prefill_gpu": True, "decode_gpu": 1}},
+ {"split_serve": {"prefill_gpu": 0, "decode_gpu": 1, "x": 1}},
+ {"split_serve": {"prefill_gpu": 0, "decode_gpu": 1, "min_prompt_tokens": "9"}},
+ {"split_serve": {"prefill_gpu": 0, "decode_gpu": 1, "prefill_args": "--x"}},
+ {"split_serve": {"prefill_gpu": 0, "decode_gpu": 1, "shared_arena": 3}}]
+ for cfg in bad:
+ with self.assertRaises(SplitConfigError, msg=cfg):
+ split_spec(cfg)
+
+ def test_check_cfg_refuses_what_cannot_run(self):
+ spec = split_spec({"split_serve": {"prefill_gpu": 0, "decode_gpu": 1}})
+ check_cfg({"args": ["--serve", "--prompt-cache", "3"]}, spec)
+ for cfg in ({"args": ["--layer-split", "24"]}, {"args": ["--batch", "2"]}, {"args": ["--prompt-cache", "0"]},
+ {"args": [], "replicas": 2}, {"args": [], "parallel": 2}, {"args": ["--peer-device", "1"]}):
+ with self.assertRaises(SplitConfigError, msg=cfg):
+ check_cfg(cfg, spec)
+ spec2 = split_spec({"split_serve": {"prefill_gpu": 0, "decode_gpu": 1, "decode_args": ["--batch", "2"]}})
+ with self.assertRaises(SplitConfigError):
+ check_cfg({"args": []}, spec2)
+
+ def test_role_cfg(self):
+ cfg = {"gpu": [0, 1], "args": ["--x"], "log": "/l/strata.log", "hip_ordinal": 1, "gpu_roles": {"0": "main"},
+ "expert_profile_save": "/p/prof.bin",
+ "split_serve": {"prefill_gpu": 2, "decode_gpu": 3, "decode_args": ["--expert-cache", "auto"]}}
+ spec = split_spec(cfg)
+ p, d = role_cfg(cfg, spec, "prefill"), role_cfg(cfg, spec, "decode")
+ self.assertEqual((p["gpu"], d["gpu"]), (2, 3))
+ self.assertEqual((p["log"], d["log"]), ("/l/strata.prefill.log", "/l/strata.decode.log"))
+ self.assertEqual(p["args"], ["--x"])
+ self.assertEqual(d["args"], ["--x", "--expert-cache", "auto"])
+ self.assertEqual(d["expert_profile_save"], "/p/prof.decode.bin")
+ for c in (p, d):
+ self.assertNotIn("split_serve", c)
+ self.assertNotIn("hip_ordinal", c)
+ self.assertNotIn("gpu_roles", c)
+ self.assertEqual(cfg["args"], ["--x"]) # the original is untouched
+
+ def test_arena_choice(self):
+ spec = split_spec({"split_serve": {"prefill_gpu": 0, "decode_gpu": 1}})
+ off = split_spec({"split_serve": {"prefill_gpu": 0, "decode_gpu": 1, "shared_arena": False}})
+ named = split_spec({"split_serve": {"prefill_gpu": 0, "decode_gpu": 1, "shared_arena": "/data/a.bin"}})
+ self.assertIsNone(sp.arena_path({"args": ["--shared-expert-arena", "/x"]}, spec)[0])
+ self.assertIsNone(sp.arena_path({"args": []}, off)[0])
+ if os.name != "nt":
+ self.assertEqual(sp.arena_path({"args": []}, named)[0], os.path.abspath("/data/a.bin"))
+ path, note = sp.arena_path({"args": ["--pack", "/no/such/pack"]}, spec) # nothing to size it from
+ self.assertIsNone(path)
+ self.assertIn("experts.bin", note)
+
+ def test_common_prefix(self):
+ self.assertEqual(common_prefix([1, 2, 3], [1, 2, 9, 4]), 2)
+ self.assertEqual(common_prefix([], [1]), 0)
+ self.assertEqual(common_prefix([1, 2], [1, 2]), 2)
+
+
+class FakeEngine:
+ """The part of StrataEngine the SplitEngine uses. `session` is the token list its session holds (a file is a json-free
+ list in a dict keyed by path, shared between the two fakes)."""
+ files = {}
+
+ def __init__(self, name, tokens_per_answer=3):
+ self.name, self.up, self.max_context, self.info, self.last = name, True, 4096, {}, {}
+ self.session, self.log, self.progress, self.prefill_tok_s_mean = [], [], None, None
+ self.answer = tokens_per_answer
+ self.fail_save = self.fail_restore = self.fail_prefill = None
+ self.unloaded = False
+
+ def alive(self):
+ return self.up
+
+ def restart(self, tries=3):
+ self.up = True
+
+ def exit_code(self):
+ return None
+
+ def death_note(self):
+ return ""
+
+ def unload(self):
+ self.up = False
+
+ def close(self):
+ self.up = False
+
+ def generate(self, ids, max_new, sampling, cancel, embeddings=None, prefill_only=False):
+ if not self.up:
+ raise EngineDied("down")
+ if prefill_only:
+ if self.fail_prefill:
+ raise self.fail_prefill
+ have = common_prefix(self.session, ids[:-1])
+ self.log.append(("prefill", len(ids)))
+ self.session = list(ids[:-1])
+ self.last = {"generated": 0, "prompt_tokens": len(ids), "prompt_ms": 1000.0, "decode_ms": 0.0,
+ "finish": "length", "reused": have, "prompt_read": len(ids) - have}
+ yield None
+ return
+ have = common_prefix(self.session, ids[:-1])
+ out = [100 + i for i in range(min(max_new, self.answer))]
+ self.log.append(("generate", len(ids), have))
+ self.session = list(ids) + out[:-1]
+ self.last = {"generated": len(out), "prompt_tokens": len(ids), "prompt_ms": 50.0, "decode_ms": 200.0,
+ "finish": "length", "reused": have, "prompt_read": len(ids) - have}
+ for t in out:
+ yield t
+
+ def session_file(self, action, path):
+ if action == "save":
+ if self.fail_save:
+ raise self.fail_save
+ FakeEngine.files[path] = list(self.session)
+ with open(path, "wb") as f:
+ f.write(b"x" * 1000)
+ return {"tokens": len(self.session), "bytes": 1000, "ms": 12.0}
+ if self.fail_restore:
+ raise self.fail_restore
+ self.session = list(FakeEngine.files[path])
+ return {"tokens": len(self.session), "bytes": 1000, "ms": 7.0}
+
+
+class Handover(unittest.TestCase):
+ def setUp(self):
+ FakeEngine.files = {}
+ self.dir = tempfile.mkdtemp(prefix="split-test-")
+ self.p, self.d = FakeEngine("p"), FakeEngine("d")
+ self.said = []
+ self.e = SplitEngine(self.p, self.d, min_tokens=100, work_dir=self.dir, say=self.said.append)
+ self.cancel = threading.Event()
+
+ def run_one(self, ids, max_new=5):
+ return [t for t in self.e.generate(ids, max_new, {}, self.cancel) if t is not None]
+
+ def test_a_long_prompt_is_handed_over(self):
+ ids = list(range(500))
+ out = self.run_one(ids)
+ self.assertEqual(out, [100, 101, 102])
+ self.assertEqual(self.p.log, [("prefill", 500)])
+ self.assertEqual(self.d.log, [("generate", 500, 499)]) # the decode engine read one token itself
+ self.assertEqual(self.e.stats["handed_over"], 1)
+ self.assertEqual(os.listdir(self.dir), []) # the session file is gone
+ last = self.e.last
+ self.assertEqual(last["reused"], 0)
+ self.assertEqual(last["prompt_read"], 500) # 499 read by the prefill engine, 1 by the decode one... 500
+ self.assertGreaterEqual(last["prompt_ms"], 12.0 + 7.0 + 50.0) # the file steps and the decode engine's start
+ self.assertLess(last["prompt_ms"], 12.0 + 7.0 + 50.0 + 1000.0)
+ self.assertEqual(last["split_bytes"], 1000)
+
+ def test_a_short_prompt_stays_on_the_decode_engine(self):
+ self.run_one(list(range(50)))
+ self.assertEqual(self.p.log, [])
+ self.assertEqual(self.e.stats["short"], 1)
+
+ def test_the_next_turn_is_not_handed_over_when_the_decode_engine_holds_it(self):
+ ids = list(range(500))
+ out = self.run_one(ids)
+ turn2 = ids + out + list(range(7000, 7040)) # the answer and a short question
+ self.run_one(turn2)
+ self.assertEqual(len(self.p.log), 1) # only the first turn went through the prefill engine
+ self.assertEqual(self.d.log[-1][0], "generate")
+ self.assertGreater(self.d.log[-1][2], 490) # and the decode engine continued its own session
+
+ def test_a_big_new_part_is_handed_over_again(self):
+ ids = list(range(500))
+ out = self.run_one(ids)
+ self.run_one(ids + out + list(range(9000, 9400)))
+ self.assertEqual(len(self.p.log), 2)
+
+ def test_pictures_go_to_the_decode_engine(self):
+ list(self.e.generate(list(range(500)), 3, {}, self.cancel, embeddings="/tmp/x"))
+ self.assertEqual(self.p.log, [])
+
+ def test_a_dead_prefill_engine_is_not_an_error(self):
+ self.p.up = False
+ self.e.heal = lambda: None
+ out = self.run_one(list(range(500)))
+ self.assertEqual(out, [100, 101, 102])
+ self.assertEqual(self.d.log, [("generate", 500, 0)]) # the decode engine read all of it
+
+ def test_a_prefill_engine_that_dies_midway_falls_back(self):
+ self.p.fail_prefill = EngineDied("gone")
+ self.e.heal = lambda: None
+ out = self.run_one(list(range(500)))
+ self.assertEqual(out, [100, 101, 102])
+ self.assertEqual(self.d.log, [("generate", 500, 0)])
+ self.assertTrue(any("failed" in s for s in self.said))
+
+ def test_a_refused_file_falls_back(self):
+ for which in ("fail_save", "fail_restore"):
+ FakeEngine.files = {}
+ self.p.session, self.d.session = [], []
+ setattr(self.p if which == "fail_save" else self.d, which, SessionRefused("invalid", "other model", False))
+ self.d.log.clear()
+ ids = list(range(500)) if which == "fail_save" else list(range(1000, 1500))
+ out = self.run_one(ids)
+ self.assertEqual(out, [100, 101, 102])
+ self.assertEqual(self.d.log, [("generate", 500, 0)], which)
+ setattr(self.p if which == "fail_save" else self.d, which, None)
+ self.assertEqual(self.e.stats["fallbacks"], 2)
+ self.assertEqual(os.listdir(self.dir), [])
+
+ def test_a_decode_engine_that_dies_in_the_restore_is_an_error(self):
+ def boom(*a):
+ self.d.up = False
+ raise EngineDied("transfer failed")
+ self.d.fail_restore = None
+ orig = self.d.session_file
+ self.d.session_file = lambda action, path: boom() if action == "restore" else orig(action, path)
+ with self.assertRaises(EngineDied):
+ self.run_one(list(range(500)))
+ self.assertEqual(os.listdir(self.dir), [])
+
+ def test_cancel_during_the_prefill_ends_quietly(self):
+ self.cancel.set()
+ orig = self.p.generate
+
+ def gen(*a, **k):
+ for t in orig(*a, **k):
+ yield t
+ self.p.generate = gen
+ out = self.run_one(list(range(500)))
+ self.assertEqual(out, [])
+ self.assertEqual(self.d.log, [])
+ self.assertEqual(self.e.held, [])
+
+ def test_attributes_come_from_the_decode_engine(self):
+ self.assertEqual(self.e.max_context, 4096)
+ self.assertEqual(self.e.batch, 0)
+ self.assertFalse(hasattr(self.e, "session_file"))
+ self.assertTrue(self.e.alive())
+ self.d.up = False
+ self.assertFalse(self.e.alive())
+ self.assertTrue(self.e.info["split_serve"])
+ self.e.silence_s = 5
+ self.assertEqual((self.p.silence_s, self.d.silence_s), (5, 5))
+
+ def test_close_removes_the_work_dir(self):
+ self.e.close()
+ self.assertFalse(os.path.exists(self.dir))
+ self.assertFalse(self.p.up or self.d.up)
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/src/program/generate.cpp b/src/program/generate.cpp
index 4e871b0d2..a16001a2b 100644
--- a/src/program/generate.cpp
+++ b/src/program/generate.cpp
@@ -12449,6 +12449,10 @@ int main(int argc, char** argv) {
// parked after it. It still resumes from a checkpoint it matches, and still saves the system-prompt root
// when that reaches --prompt-cache-root. Absent = checkpointed as before.
int req_ckpt = 1;
+ // prefill=1 (with max_new 0): split serving. Read the prompt as a request would, up to the place where the first
+ // verify window starts (the session holds n - 1 tokens, exactly what a next turn that continues it needs), and
+ // stop: no token is made. A SAVE after it is the hand-over to another engine (docs/SPLIT_SERVE.md).
+ bool req_prefill_only = false;
// pin=N: the first N prompt tokens are a shared read-only prefix (a long document that many short queries
// follow). The prompt is read in two parts at N, the checkpoint there is PINNED (retention never evicts it,
// a parked conversation holding it stays parked) and a later request that resumes from it does not park the
@@ -12479,6 +12483,7 @@ int main(int argc, char** argv) {
if (key == "logit_bias") { req_bias_text = tok.substr(eq + 1); req_bias_present = true; }
else if (key == "cvec") req_cvec = std::atoi(tok.c_str() + eq + 1);
else if (key == "ckpt") req_ckpt = std::atoi(tok.c_str() + eq + 1) != 0;
+ else if (key == "prefill") req_prefill_only = std::atoi(tok.c_str() + eq + 1) != 0;
else if (key == "pin") req_pin = std::max<long long>(0, std::atoll(tok.c_str() + eq + 1));
else if (key == "temperature") req_temperature = fv;
else if (key == "top_p") req_top_p = fv;
@@ -12524,11 +12529,16 @@ int main(int argc, char** argv) {
}
std::vector<int64_t> ids;
std::string pe;
- if (max_new < 1 || endp == nullptr || (geni && emb_path.empty()) || !parse_i64_list(endp, ids, pe)) {
+ if ((max_new < 1 && !(max_new == 0 && req_prefill_only)) || (req_prefill_only && max_new != 0) || endp == nullptr ||
+ (geni && emb_path.empty()) || !parse_i64_list(endp, ids, pe)) {
std::printf("ERR bad request: %s\n", pe.empty() ? "max_new" : pe.c_str());
continue;
}
const int64_t n = (int64_t) ids.size();
+ if (req_prefill_only && (geni || o.batch > 0 || n < 2)) {
+ std::printf("ERR prefill=1 needs a text prompt of at least 2 tokens and no --batch\n");
+ continue;
+ }
req_imgs.clear();
if (geni && !o.vision) { std::printf("ERR this engine was started without --vision\n"); continue; }
if (geni || !mrope_identity) {