Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
Split serving, first version: GEN 0 prefill=1 in the engine, serve/split.py (SplitEngine, config split_serve / --split-serve), tests
Niko1221690172c464ef
diff --git a/serve/server.py b/serve/server.pyindex 76b5a2c79..8fa45c4d4 100644--- a/serve/server.py+++ b/serve/server.py@@ -2183,7 +2183,7 @@ class StrataEngine:"tok_s": round(r["generated"] / max(1e-6, now - ft), 1) if ft else None})return view- def generate(self, ids, max_new, sampling, cancel, embeddings=None):+ def generate(self, ids, max_new, sampling, cancel, embeddings=None, prefill_only=False):"""Yields token ids, and None as a heartbeat every 10 s while the engine is quiet (reading a long prompt):the HTTP layer turns it into an SSE comment, which keeps clients' watchdogs calm and notices a client thathas gone. A consumer that stops early (or `cancel`) makes the engine STOP, so it does not run to max_new."""@@ -2198,6 +2198,10 @@ class StrataEngine:# an image request takes the same sampling keys as text (#75: it used to decode greedily whatever was asked)head = f"GENI {int(max_new)}{self.sampling_keys(sampling or {})} {embeddings}" if embeddings else \f"GEN {int(max_new)}{self.sampling_keys(sampling or {})}"+ if prefill_only: # split serving (serve/split.py): read the prompt, make no token+ if embeddings or int(max_new) != 0:+ raise ValueError("prefill_only is a text prompt and max_new 0")+ head = "GEN 0 prefill=1"try:self.proc.stdin.write(f"{head} {','.join(str(int(t)) for t in ids)}\n")self.proc.stdin.flush()@@ -4639,6 +4643,8 @@ class Service:live["state"] = "generating"if hasattr(self.engine, "replica_view"): # opt-in replicas: each one's state and requests routed to itlive["replicas"] = self.engine.replica_view()+ if hasattr(self.engine, "split_view"): # opt-in split serving: the hand-overs so far (serve/split.py)+ live["split_serve"] = self.engine.split_view()engine = {"model": self.model, "max_context": self.reported_ctx(), "images": self.vision is not None,**dict(getattr(self.engine, "info", {}) or {})}tel = self.telemetry.snapshot() if getattr(self, "telemetry", None) else {"now": {}, "history": {}, "static": {}}@@ -8008,6 +8014,39 @@ def start_replicas(cfg: dict, groups: list[list[int]], exe: str, effort_end: lisreturn rp.ReplicaEngine(engines, load_tokens=load)+def start_split(cfg: dict, spec: dict, exe: str, effort_end: list[str], lazy: bool, *, offline: bool = False):+ """Opt-in split serving (serve/split.py): a prefill engine and a decode engine, one card each, the same model and the+ same session settings. Started one after the other, so a shared expert arena is filled once; the SplitEngine in+ front hands a long prompt from the first to the second."""+ from serve import split as sp+ sp.check_cfg(cfg, spec)+ arena, arena_note = sp.arena_path(cfg, spec)+ work = sp.session_dir(spec)+ print(f"[strata] split serving: the prompt is read on card {spec['prefill_gpu']}, the answer is made on card "+ f"{spec['decode_gpu']} (prompts with {spec['min_prompt_tokens']}+ new tokens are handed over); {arena_note}",+ flush=True)+ engines = {}+ try:+ for role in ("prefill", "decode"):+ rc = sp.role_cfg(cfg, spec, role)+ if arena:+ rc["args"] = list(rc["args"]) + ["--shared-expert-arena", arena]+ if role == "prefill":+ warn_budget_over_ram(engine_args(rc) if "args" in rc else [])+ engines[role] = StrataEngine(exe, engine_args(rc) + effort_end, cwd=rc.get("cwd"), log=rc.get("log"),+ env=child_env(rc, offline=offline), lazy=lazy)+ except BaseException:+ for e in engines.values():+ try:+ e.close()+ except Exception: # noqa: BLE001+ pass+ shutil.rmtree(work, ignore_errors=True)+ raise+ return sp.SplitEngine(engines["prefill"], engines["decode"], min_tokens=spec["min_prompt_tokens"], work_dir=work,+ arena_file=arena)++def check_request_sampling(req) -> None:"""A request's sampling field of the wrong JSON type ("temperature": "0.7", "top_k": 40.0, "seed": true) is a 400that names the field, as the same value is through POST /settings or the run config. Unchecked, the value@@ -8211,6 +8250,10 @@ def main(argv=None, *, lifecycle_event=None, interrupt_exit_code: int = 0) -> inhelp="data-parallel replicas (opt-in): cut the \"gpu\" list into N equal groups and run one engine on ""each, requests spread over them (also \"replicas\" in the config: a number, or ""[{\"gpus\": [0, 1]}, {\"gpus\": [2, 3]}])")+ ap.add_argument("--split-serve", default=None, metavar="PREFILL,DECODE",+ help="split serving (opt-in): one engine on card PREFILL reads the prompt, a second on card DECODE makes "+ "the answer, the session handed over in RAM (also \"split_serve\" in the config: "+ "{\"prefill_gpu\": 0, \"decode_gpu\": 1}; docs/SPLIT_SERVE.md)")launch_args = list(sys.argv[1:] if argv is None else argv)a = ap.parse_args(launch_args)if a.engine == "strata" and (_rt := rttime_limit_note()): # #3238: say it before the first unexplained -9@@ -8284,6 +8327,12 @@ def main(argv=None, *, lifecycle_event=None, interrupt_exit_code: int = 0) -> inlift_reasoning_effort_arg(cfg)if a.replicas is not None:cfg["replicas"] = a.replicas+ if a.split_serve is not None:+ pair = [x.strip() for x in a.split_serve.split(",")]+ if len(pair) != 2 or not all(x.isdecimal() for x in pair):+ ap.error("--split-serve takes two card numbers: PREFILL,DECODE (for example 0,1)")+ cfg["split_serve"] = {**(cfg.get("split_serve") if isinstance(cfg.get("split_serve"), dict) else {}),+ "prefill_gpu": int(pair[0]), "decode_gpu": int(pair[1])}if a.gpu is not None:cfg["gpu"] = int(a.gpu) if a.gpu.strip().isdigit() else a.gpua.host = a.host or cfg.get("host") or "127.0.0.1" # issue #26: the run scripts pass no --host, the config can@@ -8412,15 +8461,24 @@ def main(argv=None, *, lifecycle_event=None, interrupt_exit_code: int = 0) -> ingroups = rp.gpu_groups(cfg)except rp.ReplicaConfigError as e:raise SystemExit(f"[strata] config {e}")+ from serve import split as split_mod+ try:+ split = split_mod.split_spec(cfg)+ if split:+ split_mod.check_cfg(cfg, split)+ except split_mod.SplitConfigError as e:+ raise SystemExit(f"[strata] config {e}")if groups:engine = start_replicas(cfg, groups, exe, effort_end or [], lazy, offline=a.offline)+ elif split:+ engine = start_split(cfg, split, exe, effort_end or [], lazy, offline=a.offline)else:engine = StrataEngine(exe, engine_args(cfg) + (effort_end or []), cwd=cfg.get("cwd"), log=cfg.get("log"),env=env, lazy=lazy)engine.silence_s = silence # an attribute of its own: restart() keeps itwarn_tight_ram(engine.info.get("arena_mib"))note = desktop_vram_note(cfg.get("backend"), engine.info.get("vram_free_mib"), engine.spawn[1],- linux_desktop()) if not groups else None+ linux_desktop()) if not groups and not split else Noneif note: # #560 #516: before --open starts a browser on that cardprint(note, flush=True)else:@@ -8588,8 +8646,12 @@ def main(argv=None, *, lifecycle_event=None, interrupt_exit_code: int = 0) -> insvc.gpu_indices = [c for g in groups for c in g]svc.gpu_index = svc.gpu_indices[0]svc.replica_gpu_groups = groups+ if a.engine == "strata" and split: # split serving: the two cards (the prefill card first)+ svc.gpu_indices = [split["prefill_gpu"], split["decode_gpu"]]+ svc.gpu_index = split["decode_gpu"]svc.terminal_visibility_order = visible_device_order(env, cfg.get("backend")) if a.engine == "strata" else []svc.terminal_topology_mode = ("replicas (configured)" if a.engine == "strata" and groups+ else "split serve (configured)" if a.engine == "strata" and splitelse "layer split (configured)" if a.engine == "strata" and layer_split_of(cfg)else "single GPU or CPU" if a.engine == "strata" else "mock")svc.terminal_configured_split = (layer_split_value(cfg) if a.engine == "strata" and layer_split_of(cfg)diff --git a/serve/split.py b/serve/split.pynew file mode 100644index 000000000..5b28394a1--- /dev/null+++ b/serve/split.py@@ -0,0 +1,488 @@+"""Split serving: one engine reads the prompt, another makes the answer (issue #4000, idea and manual setup by @MarcoTool).++Two engines on the same PC, one card each, the same model files and the same session settings. A long prompt is read by+the prefill engine, which saves the session (the engine's own session file: SAVE <path>) into RAM; the decode engine+restores it (RESTORE <path>) and makes the answer. The server keeps its one HTTP front, its one Service and its one set of+endpoints; `SplitEngine` stands in for the engine, as `ReplicaEngine` does for replicas.++Opt-in, in the run config (nothing changes without it):++ "split_serve": {"prefill_gpu": 0, "decode_gpu": 1}++ prefill_gpu / decode_gpu the card of each engine, numbered as nvidia-smi numbers them (required, different)+ min_prompt_tokens hand a request over only when the decode engine would read at least this many new tokens+ itself (default 2048); a shorter tail is read by the decode engine in place+ dir where the session files go (default /dev/shm when it has room, else the temp folder)+ shared_arena true (default): one copy of the experts in RAM for both engines (Linux, a file in /dev/shm+ when the experts fit there); false: each engine keeps its own copy; a path: that file+ prefill_args / decode_args extra engine arguments for one role (a bigger expert cache for the decode engine, say)++What must match between the two engines is what the session file checks (docs/SPLIT_SERVE.md): the model files, the engine+version, the backend, --kv, --max-context, --kv-resident, --mtp-window, the rope settings and the arithmetic switches. They+start from one config, so they do. Every failure on the way (a prefill engine that died, a refused file) is not an error+for the client: the decode engine reads the prompt itself, as it would without the option.++Honest limit of this first version: one request at a time (the Service's FIFO), one conversation resident on the decode+engine (a restore replaces what it held), no parallel slots (--batch), no layer split in an engine, no pictures (a request+with pictures goes to the decode engine alone).+"""+from __future__ import annotations++import os+import shutil+import tempfile+import threading+import time++MIN_TOKENS = 2048 # the decode engine's own tail below this is read in place+KNOWN_KEYS = {"prefill_gpu", "decode_gpu", "min_prompt_tokens", "dir", "shared_arena", "prefill_args", "decode_args"}+FORBIDDEN_ARGS = { # engine arguments that cannot be used in a split-serve engine, and why+ "--layer-split": "an engine of the split runs on one card",+ "--split-device": "an engine of the split runs on one card",+ "--peer-device": "session files do not support --peer-device",+ "--batch": "session files do not support --batch (parallel requests)",+ "--slots": "session files do not support parallel slots",+}+HEAL_COOLDOWN_S = 60.0 # a prefill engine that died is started again at most this often+HEARTBEAT_S = 5.0 # a long file step sends a keep-alive this often+++class SplitConfigError(ValueError):+ pass+++def split_spec(cfg: dict) -> dict | None:+ """The checked "split_serve" section of the run config, or None when the option is off."""+ raw = cfg.get("split_serve")+ if raw is None or raw is False:+ return None+ if not isinstance(raw, dict):+ raise SplitConfigError('"split_serve" must be {"prefill_gpu": N, "decode_gpu": M}, not ' + repr(raw))+ unknown = sorted(set(raw) - KNOWN_KEYS)+ if unknown:+ raise SplitConfigError(f'"split_serve" has no key {unknown[0]!r} (keys: {", ".join(sorted(KNOWN_KEYS))})')+ cards = []+ for k in ("prefill_gpu", "decode_gpu"):+ v = raw.get(k)+ if isinstance(v, str) and v.strip().isdigit():+ v = int(v.strip())+ if isinstance(v, bool) or not isinstance(v, int) or v < 0:+ raise SplitConfigError(f'"split_serve" needs "{k}": the number of a card (0, 1, ...), not {v!r}')+ cards.append(v)+ if cards[0] == cards[1]:+ raise SplitConfigError(f'"split_serve": prefill_gpu and decode_gpu are both card {cards[0]}; each engine needs a '+ "card of its own")+ out = {"prefill_gpu": cards[0], "decode_gpu": cards[1]}+ n = raw.get("min_prompt_tokens", MIN_TOKENS)+ if isinstance(n, bool) or not isinstance(n, int) or n < 0:+ raise SplitConfigError(f'"split_serve": "min_prompt_tokens" must be a number of tokens (0 or more), not {n!r}')+ out["min_prompt_tokens"] = n+ d = raw.get("dir")+ if d is not None and (not isinstance(d, str) or not d.strip()):+ raise SplitConfigError(f'"split_serve": "dir" must be a folder name, not {d!r}')+ out["dir"] = d+ sa = raw.get("shared_arena", True)+ if not (isinstance(sa, bool) or (isinstance(sa, str) and sa.strip())):+ raise SplitConfigError(f'"split_serve": "shared_arena" must be true, false or a file name, not {sa!r}')+ out["shared_arena"] = sa+ for k in ("prefill_args", "decode_args"):+ a = raw.get(k, [])+ if not isinstance(a, list) or not all(isinstance(x, (str, int, float)) and not isinstance(x, bool) for x in a):+ raise SplitConfigError(f'"split_serve": "{k}" must be a list of engine arguments')+ out[k] = [str(x) for x in a]+ return out+++def check_cfg(cfg: dict, spec: dict) -> None:+ """Refuse (with the reason, in plain words) a config the split cannot run."""+ if cfg.get("replicas") not in (None, False):+ raise SplitConfigError('"split_serve" and "replicas" cannot be used together: both start engines on the cards')+ args = [str(a) for a in cfg.get("args") or []] + spec["prefill_args"] + spec["decode_args"]+ for flag, why in FORBIDDEN_ARGS.items():+ if flag in args:+ raise SplitConfigError(f'"split_serve" cannot be used with {flag} in the engine arguments: {why}')+ for i, a in enumerate(args):+ if a == "--prompt-cache" and i + 1 < len(args) and args[i + 1].strip() == "0":+ raise SplitConfigError('"split_serve" needs the conversation cache (--prompt-cache 1 or more): the session '+ "file is made from it")+ if cfg.get("batch") or cfg.get("parallel"):+ raise SplitConfigError('"split_serve" does not run parallel requests ("parallel"/"batch"): remove it')+++def _suffixed(path, tag: str):+ if not isinstance(path, str) or not path:+ return path+ root, ext = os.path.splitext(path)+ return f"{root}.{tag}{ext}"+++def role_cfg(cfg: dict, spec: dict, role: str) -> dict:+ """The config of one engine of the split: its own card, log and learned-profile file; its role's extra arguments."""+ import copy+ c = copy.deepcopy(cfg)+ c.pop("split_serve", None)+ c["gpu"] = spec[f"{role}_gpu"]+ c.pop("hip_ordinal", None)+ c.pop("gpu_roles", None) # the roles were for a layer split over several cards+ c["log"] = _suffixed(cfg.get("log"), role)+ if cfg.get("expert_profile_save"):+ c["expert_profile_save"] = _suffixed(cfg["expert_profile_save"], role)+ extra = spec[f"{role}_args"]+ if extra:+ c["args"] = list(c["args"]) + extra+ return c+++def _arg_value(args: list, flag: str):+ for i, a in enumerate(args):+ if a == flag and i + 1 < len(args):+ return args[i + 1]+ return None+++def arena_path(cfg: dict, spec: dict) -> tuple[str | None, str]:+ """(file for --shared-expert-arena or None, a line for the log). Only Linux can share the arena; the experts+ file must fit /dev/shm (a tmpfs is half of the RAM by default) unless the config names another file."""+ args = [str(a) for a in cfg.get("args") or []]+ if "--shared-expert-arena" in args:+ return None, "the config already names a shared expert arena"+ sa = spec["shared_arena"]+ if sa is False:+ return None, "shared_arena is off: each engine keeps its own copy of the experts in RAM"+ if os.name == "nt":+ return None, ("this OS cannot share the expert arena between two engines (Linux only): each engine keeps its own "+ "copy of the experts in RAM, so the PC needs RAM for two")+ if isinstance(sa, str):+ return os.path.abspath(sa), f"shared expert arena in {os.path.abspath(sa)}"+ pack = _arg_value(args, "--pack")+ experts = os.path.join(pack, "experts.bin") if pack else None+ if not experts or not os.path.isfile(experts):+ return None, ("no experts.bin to size a shared arena from (a pack that reads the GGUF in place): each engine "+ "keeps its own copy of the experts; name a file in \"shared_arena\" to share one")+ need = os.path.getsize(experts)+ try:+ free = shutil.disk_usage("/dev/shm").free+ except OSError:+ free = 0+ if free < need + (1 << 30):+ return None, (f"/dev/shm has {free / 2**30:.1f} GiB free and the experts are {need / 2**30:.1f} GiB: each engine "+ "keeps its own copy of the experts in RAM (two copies). To share one copy name a file in "+ "\"shared_arena\" (a file on disk works, it is written once) or make /dev/shm bigger")+ return "/dev/shm/strata-split-arena.bin", f"one shared expert arena in /dev/shm ({need / 2**30:.1f} GiB)"+++def session_dir(spec: dict) -> str:+ d = spec.get("dir")+ if d:+ os.makedirs(d, exist_ok=True)+ return d+ shm = "/dev/shm"+ try:+ if os.name != "nt" and os.path.isdir(shm) and os.access(shm, os.W_OK) and shutil.disk_usage(shm).free > (6 << 30):+ return tempfile.mkdtemp(prefix="strata-split-", dir=shm)+ except OSError:+ pass+ return tempfile.mkdtemp(prefix="strata-split-")+++def common_prefix(a, b) -> int:+ n = min(len(a), len(b))+ i = 0+ while i < n and a[i] == b[i]:+ i += 1+ return i+++class SplitEngine:+ """Stands in for one engine (the interface Service uses), over a prefill engine and a decode engine."""++ NOT_HERE = {"session_file", "vram", "ctl", "slot_cv"} # not offered: one conversation file / VRAM resize / control line+ PROPAGATE = {"silence_s", "gpu_busy"}++ def __init__(self, prefill, decode, *, min_tokens: int = MIN_TOKENS, work_dir: str | None = None,+ arena_file: str | None = None, say=print):+ d = self.__dict__+ d["prefill"], d["decode"] = prefill, decode+ d["cur"] = decode # the engine the request is on now (progress, last)+ d["min_tokens"] = int(min_tokens)+ d["work_dir"] = work_dir or tempfile.mkdtemp(prefix="strata-split-")+ d["arena_file"] = arena_file+ d["say"] = say+ d["held"] = [] # the tokens the decode engine's session holds (an estimate: ids + answer)+ d["seq"] = 0+ d["heal_at"] = 0.0+ d["healing"] = False+ d["lock"] = threading.Lock()+ d["stats"] = {"requests": 0, "handed_over": 0, "fallbacks": 0, "short": 0, "bytes": 0, "prefill_ms": 0.0,+ "save_ms": 0.0, "restore_ms": 0.0, "last": None}++ # ---- attribute plumbing+ def __getattr__(self, name):+ if name in self.NOT_HERE or name.startswith("__"):+ raise AttributeError(name)+ return getattr(self.decode, name)++ def __setattr__(self, name, value):+ if name in self.PROPAGATE:+ self.prefill.__setattr__(name, value)+ self.decode.__setattr__(name, value)+ else:+ self.__dict__[name] = value++ @property+ def last(self):+ return self.decode.last++ @last.setter+ def last(self, value):+ self.decode.last = value++ @property+ def progress(self):+ return self.cur.progress++ @property+ def prefill_tok_s_mean(self):+ return self.cur.prefill_tok_s_mean++ @property+ def batch(self) -> int:+ return 0++ @property+ def max_context(self) -> int:+ v = [e.max_context for e in (self.prefill, self.decode) if getattr(e, "max_context", 0) and e.alive()]+ return min(v) if v else 0++ @property+ def unloaded(self) -> bool:+ return all(getattr(e, "unloaded", False) for e in (self.prefill, self.decode))++ @property+ def starting(self) -> bool:+ return not self.alive() and any(getattr(e, "starting", False) for e in (self.prefill, self.decode))++ @property+ def info(self) -> dict:+ i = dict(self.decode.info)+ i["split_serve"] = True+ i["split_prefill_alive"] = bool(self.prefill.alive())+ return i++ @info.setter+ def info(self, value):+ pass++ def alive(self) -> bool:+ return bool(self.decode.alive())++ def exit_code(self):+ return next((c for c in (e.exit_code() for e in (self.decode, self.prefill) if not e.alive()) if c is not None),+ None)++ def death_note(self) -> str:+ return next((e.death_note() for e in (self.decode, self.prefill) if not e.alive()), "")++ def split_view(self) -> dict:+ """For /metrics: how the two engines are used."""+ s = dict(self.stats)+ s.update(prefill_alive=bool(self.prefill.alive()), decode_alive=bool(self.decode.alive()),+ min_prompt_tokens=self.min_tokens)+ return s++ # ---- life cycle+ def restart(self, tries: int = 3):+ errors = {}+ for name, e in (("decode", self.decode), ("prefill", self.prefill)): # one at a time: the arena fills once+ if e.alive():+ continue+ try:+ e.restart(tries)+ except BaseException as x: # noqa: BLE001+ errors[name] = x+ if "decode" in errors:+ raise errors["decode"]+ for name, x in errors.items():+ self.say(f"[strata] the {name} engine of the split did not start ({x}); the other one serves")+ self.__dict__["held"] = []++ def unload(self):+ for e in (self.prefill, self.decode):+ e.unload()+ self.__dict__["held"] = []++ def close(self):+ err = None+ for e in (self.prefill, self.decode):+ try:+ e.close()+ except BaseException as x: # noqa: BLE001+ err = err or x+ shutil.rmtree(self.work_dir, ignore_errors=True)+ if self.arena_file and self.arena_file.startswith("/dev/shm/strata-split-"):+ try:+ os.unlink(self.arena_file) # RAM until it is gone+ except OSError:+ pass+ if err:+ raise err++ def heal(self) -> None:+ """Start a prefill engine that died again, in the background (the decode engine serves meanwhile)."""+ e = self.prefill+ now = time.monotonic()+ if e.alive() or getattr(e, "unloaded", False) or getattr(e, "starting", False) or self.healing or now < self.heal_at:+ return+ self.__dict__["heal_at"] = now + HEAL_COOLDOWN_S+ self.__dict__["healing"] = True++ def run():+ try:+ self.say("[strata] split: the prefill engine had stopped; starting it again")+ e.restart()+ self.say("[strata] split: the prefill engine is running again")+ except BaseException as x: # noqa: BLE001+ self.say(f"[strata] split: the prefill engine did not start again: {x}")+ finally:+ self.__dict__["healing"] = False+ threading.Thread(target=run, daemon=True).start()++ # ---- the request+ def wants_handover(self, ids) -> tuple[bool, int]:+ """(hand the prompt to the prefill engine?, the tokens the decode engine would read itself)."""+ n = len(ids)+ if n < 2:+ return False, 0+ need = (n - 1) - min(common_prefix(self.held, ids), n - 1)+ return need >= max(1, self.min_tokens) and self.prefill.alive(), need++ def _wait(self, fn):+ """Run a blocking engine call on a thread; yields None as a keep-alive while it runs. -> its result."""+ box = {}++ def run():+ try:+ box["r"] = fn()+ except BaseException as e: # noqa: BLE001+ box["e"] = e+ th = threading.Thread(target=run, daemon=True)+ th.start()+ while th.is_alive():+ th.join(HEARTBEAT_S)+ if th.is_alive():+ yield None+ if "e" in box:+ raise box["e"]+ return box.get("r")++ def _hand_over(self, ids, cancel, rec: dict):+ """Prefill engine reads the prompt, saves the session; the decode engine restores it. Yields keep-alives.+ rec['ok'] is True when the decode engine now holds the prompt's first n-1 tokens."""+ from serve.server import EngineDied, SessionRefused+ P, D = self.prefill, self.decode+ n = len(ids)+ self.__dict__["seq"] += 1+ path = os.path.join(self.work_dir, f"handover-{os.getpid()}-{self.seq}.session")+ rec.update(ok=False)+ t0 = time.monotonic()+ self.__dict__["cur"] = P+ try:+ try:+ for _ in P.generate(ids, 0, {}, cancel, prefill_only=True):+ yield None+ except (EngineDied, ValueError) as e:+ self.say(f"[strata] split: the prefill engine failed ({e}); the decode engine reads the prompt itself")+ self.heal()+ return+ if cancel.is_set():+ return+ p_last = dict(P.last or {})+ rec["prefill_ms"] = (time.monotonic() - t0) * 1000.0+ rec["p_last"] = p_last+ try:+ saved = yield from self._wait(lambda: P.session_file("save", path))+ self.__dict__["cur"] = D+ restored = yield from self._wait(lambda: D.session_file("restore", path))+ except SessionRefused as e:+ self.say(f"[strata] split: the session file was refused ({e}); the decode engine reads the prompt itself")+ return+ except EngineDied as e:+ if not D.alive():+ raise+ self.say(f"[strata] split: the prefill engine failed while saving ({e}); the decode engine reads the prompt "+ "itself")+ self.heal()+ return+ rec.update(save_ms=float(saved["ms"]), restore_ms=float(restored["ms"]), bytes=int(saved["bytes"]),+ tokens=int(restored["tokens"]), ok=restored["tokens"] == n - 1)+ if not rec["ok"]:+ self.say(f"[strata] split: the decode engine restored {restored['tokens']} tokens, not the {n - 1} the "+ "prompt needs; it reads the rest itself")+ rec["ok"] = restored["tokens"] > 0+ finally:+ self.__dict__["cur"] = D+ try:+ os.unlink(path)+ except OSError:+ pass++ def generate(self, ids, max_new, sampling, cancel, embeddings=None):+ D = self.decode+ ids = list(ids)+ st = self.stats+ st["requests"] += 1+ rec: dict = {}+ use, need = (False, 0) if embeddings is not None else self.wants_handover(ids)+ if not use:+ st["short"] += 1+ if use:+ if not self.prefill.alive():+ self.heal()+ yield from self._hand_over(ids, cancel, rec)+ if cancel.is_set():+ self.__dict__["held"] = []+ return+ if rec.get("ok"):+ st["handed_over"] += 1+ st["bytes"] += rec["bytes"]+ st["prefill_ms"] += rec["prefill_ms"]+ st["save_ms"] += rec["save_ms"]+ st["restore_ms"] += rec["restore_ms"]+ self.__dict__["held"] = ids[:len(ids) - 1]+ else:+ st["fallbacks"] += 1+ self.__dict__["cur"] = D+ produced: list[int] = []+ ok = False+ try:+ gen = D.generate(ids, max_new, sampling, cancel, embeddings=embeddings) if embeddings is not None \+ else D.generate(ids, max_new, sampling, cancel)+ for t in gen:+ if t is not None:+ produced.append(t)+ yield t+ ok = True+ finally:+ self.__dict__["held"] = (ids + produced) if ok else []+ if rec.get("ok") and ok:+ self._account(ids, rec)++ def _account(self, ids, rec: dict) -> None:+ """The figures of the engine's last DONE, as the whole request: the prefill engine's read, the file steps and the+ decode engine's own start are the prompt time; what the prefill engine reused is the cached part."""+ D = self.decode+ last = dict(D.last or {})+ p = rec.get("p_last") or {}+ total_prompt_ms = rec["prefill_ms"] + rec["save_ms"] + rec["restore_ms"] + float(last.get("prompt_ms") or 0.0)+ p_read = p.get("prompt_read")+ if p_read is None:+ p_read = len(ids) - int(p.get("reused") or 0)+ last.update(prompt_ms=total_prompt_ms, reused=int(p.get("reused") or 0),+ prompt_read=max(0, min(len(ids), int(p_read) + int(last.get("prompt_read") or 0))),+ split_prefill_ms=round(rec["prefill_ms"], 1), split_save_ms=round(rec["save_ms"], 1),+ split_restore_ms=round(rec["restore_ms"], 1), split_bytes=rec["bytes"])+ D.last = last+ self.stats["last"] = {k: last[k] for k in last if k.startswith("split_")}+ self.say(f"[strata] split: prompt {len(ids)} tokens read on the prefill card in {rec['prefill_ms'] / 1000:.1f} s, "+ f"session {rec['bytes'] / 2**20:.0f} MiB saved in {rec['save_ms'] / 1000:.2f} s and restored in "+ f"{rec['restore_ms'] / 1000:.2f} s; the decode card makes the answer")diff --git a/serve/test_split.py b/serve/test_split.pynew file mode 100644index 000000000..a12164363--- /dev/null+++ b/serve/test_split.py@@ -0,0 +1,271 @@+"""Split serving (serve/split.py): the config, the decision, the hand-over with two fake engines, and every fallback.+The engine-side proof (the continuation after a hand-over equals a single engine's) is tests/split_serve_exact.py."""+import os+import tempfile+import threading+import unittest++from serve import split as sp+from serve.server import EngineDied, SessionRefused+from serve.split import SplitConfigError, SplitEngine, check_cfg, common_prefix, role_cfg, split_spec+++class Config(unittest.TestCase):+ def test_off_without_the_key(self):+ self.assertIsNone(split_spec({}))+ self.assertIsNone(split_spec({"split_serve": False}))+ self.assertIsNone(split_spec({"split_serve": None}))++ def test_cards_and_defaults(self):+ s = split_spec({"split_serve": {"prefill_gpu": 0, "decode_gpu": "1"}})+ self.assertEqual((s["prefill_gpu"], s["decode_gpu"]), (0, 1))+ self.assertEqual(s["min_prompt_tokens"], sp.MIN_TOKENS)+ self.assertIs(s["shared_arena"], True)+ self.assertEqual((s["prefill_args"], s["decode_args"]), ([], []))++ def test_mistakes_are_said(self):+ bad = [{"split_serve": 1}, {"split_serve": {"prefill_gpu": 0}}, {"split_serve": {"prefill_gpu": 1, "decode_gpu": 1}},+ {"split_serve": {"prefill_gpu": 0, "decode_gpu": -1}}, {"split_serve": {"prefill_gpu": True, "decode_gpu": 1}},+ {"split_serve": {"prefill_gpu": 0, "decode_gpu": 1, "x": 1}},+ {"split_serve": {"prefill_gpu": 0, "decode_gpu": 1, "min_prompt_tokens": "9"}},+ {"split_serve": {"prefill_gpu": 0, "decode_gpu": 1, "prefill_args": "--x"}},+ {"split_serve": {"prefill_gpu": 0, "decode_gpu": 1, "shared_arena": 3}}]+ for cfg in bad:+ with self.assertRaises(SplitConfigError, msg=cfg):+ split_spec(cfg)++ def test_check_cfg_refuses_what_cannot_run(self):+ spec = split_spec({"split_serve": {"prefill_gpu": 0, "decode_gpu": 1}})+ check_cfg({"args": ["--serve", "--prompt-cache", "3"]}, spec)+ for cfg in ({"args": ["--layer-split", "24"]}, {"args": ["--batch", "2"]}, {"args": ["--prompt-cache", "0"]},+ {"args": [], "replicas": 2}, {"args": [], "parallel": 2}, {"args": ["--peer-device", "1"]}):+ with self.assertRaises(SplitConfigError, msg=cfg):+ check_cfg(cfg, spec)+ spec2 = split_spec({"split_serve": {"prefill_gpu": 0, "decode_gpu": 1, "decode_args": ["--batch", "2"]}})+ with self.assertRaises(SplitConfigError):+ check_cfg({"args": []}, spec2)++ def test_role_cfg(self):+ cfg = {"gpu": [0, 1], "args": ["--x"], "log": "/l/strata.log", "hip_ordinal": 1, "gpu_roles": {"0": "main"},+ "expert_profile_save": "/p/prof.bin",+ "split_serve": {"prefill_gpu": 2, "decode_gpu": 3, "decode_args": ["--expert-cache", "auto"]}}+ spec = split_spec(cfg)+ p, d = role_cfg(cfg, spec, "prefill"), role_cfg(cfg, spec, "decode")+ self.assertEqual((p["gpu"], d["gpu"]), (2, 3))+ self.assertEqual((p["log"], d["log"]), ("/l/strata.prefill.log", "/l/strata.decode.log"))+ self.assertEqual(p["args"], ["--x"])+ self.assertEqual(d["args"], ["--x", "--expert-cache", "auto"])+ self.assertEqual(d["expert_profile_save"], "/p/prof.decode.bin")+ for c in (p, d):+ self.assertNotIn("split_serve", c)+ self.assertNotIn("hip_ordinal", c)+ self.assertNotIn("gpu_roles", c)+ self.assertEqual(cfg["args"], ["--x"]) # the original is untouched++ def test_arena_choice(self):+ spec = split_spec({"split_serve": {"prefill_gpu": 0, "decode_gpu": 1}})+ off = split_spec({"split_serve": {"prefill_gpu": 0, "decode_gpu": 1, "shared_arena": False}})+ named = split_spec({"split_serve": {"prefill_gpu": 0, "decode_gpu": 1, "shared_arena": "/data/a.bin"}})+ self.assertIsNone(sp.arena_path({"args": ["--shared-expert-arena", "/x"]}, spec)[0])+ self.assertIsNone(sp.arena_path({"args": []}, off)[0])+ if os.name != "nt":+ self.assertEqual(sp.arena_path({"args": []}, named)[0], os.path.abspath("/data/a.bin"))+ path, note = sp.arena_path({"args": ["--pack", "/no/such/pack"]}, spec) # nothing to size it from+ self.assertIsNone(path)+ self.assertIn("experts.bin", note)++ def test_common_prefix(self):+ self.assertEqual(common_prefix([1, 2, 3], [1, 2, 9, 4]), 2)+ self.assertEqual(common_prefix([], [1]), 0)+ self.assertEqual(common_prefix([1, 2], [1, 2]), 2)+++class FakeEngine:+ """The part of StrataEngine the SplitEngine uses. `session` is the token list its session holds (a file is a json-free+ list in a dict keyed by path, shared between the two fakes)."""+ files = {}++ def __init__(self, name, tokens_per_answer=3):+ self.name, self.up, self.max_context, self.info, self.last = name, True, 4096, {}, {}+ self.session, self.log, self.progress, self.prefill_tok_s_mean = [], [], None, None+ self.answer = tokens_per_answer+ self.fail_save = self.fail_restore = self.fail_prefill = None+ self.unloaded = False++ def alive(self):+ return self.up++ def restart(self, tries=3):+ self.up = True++ def exit_code(self):+ return None++ def death_note(self):+ return ""++ def unload(self):+ self.up = False++ def close(self):+ self.up = False++ def generate(self, ids, max_new, sampling, cancel, embeddings=None, prefill_only=False):+ if not self.up:+ raise EngineDied("down")+ if prefill_only:+ if self.fail_prefill:+ raise self.fail_prefill+ have = common_prefix(self.session, ids[:-1])+ self.log.append(("prefill", len(ids)))+ self.session = list(ids[:-1])+ self.last = {"generated": 0, "prompt_tokens": len(ids), "prompt_ms": 1000.0, "decode_ms": 0.0,+ "finish": "length", "reused": have, "prompt_read": len(ids) - have}+ yield None+ return+ have = common_prefix(self.session, ids[:-1])+ out = [100 + i for i in range(min(max_new, self.answer))]+ self.log.append(("generate", len(ids), have))+ self.session = list(ids) + out[:-1]+ self.last = {"generated": len(out), "prompt_tokens": len(ids), "prompt_ms": 50.0, "decode_ms": 200.0,+ "finish": "length", "reused": have, "prompt_read": len(ids) - have}+ for t in out:+ yield t++ def session_file(self, action, path):+ if action == "save":+ if self.fail_save:+ raise self.fail_save+ FakeEngine.files[path] = list(self.session)+ with open(path, "wb") as f:+ f.write(b"x" * 1000)+ return {"tokens": len(self.session), "bytes": 1000, "ms": 12.0}+ if self.fail_restore:+ raise self.fail_restore+ self.session = list(FakeEngine.files[path])+ return {"tokens": len(self.session), "bytes": 1000, "ms": 7.0}+++class Handover(unittest.TestCase):+ def setUp(self):+ FakeEngine.files = {}+ self.dir = tempfile.mkdtemp(prefix="split-test-")+ self.p, self.d = FakeEngine("p"), FakeEngine("d")+ self.said = []+ self.e = SplitEngine(self.p, self.d, min_tokens=100, work_dir=self.dir, say=self.said.append)+ self.cancel = threading.Event()++ def run_one(self, ids, max_new=5):+ return [t for t in self.e.generate(ids, max_new, {}, self.cancel) if t is not None]++ def test_a_long_prompt_is_handed_over(self):+ ids = list(range(500))+ out = self.run_one(ids)+ self.assertEqual(out, [100, 101, 102])+ self.assertEqual(self.p.log, [("prefill", 500)])+ self.assertEqual(self.d.log, [("generate", 500, 499)]) # the decode engine read one token itself+ self.assertEqual(self.e.stats["handed_over"], 1)+ self.assertEqual(os.listdir(self.dir), []) # the session file is gone+ last = self.e.last+ self.assertEqual(last["reused"], 0)+ self.assertEqual(last["prompt_read"], 500) # 499 read by the prefill engine, 1 by the decode one... 500+ self.assertGreaterEqual(last["prompt_ms"], 12.0 + 7.0 + 50.0) # the file steps and the decode engine's start+ self.assertLess(last["prompt_ms"], 12.0 + 7.0 + 50.0 + 1000.0)+ self.assertEqual(last["split_bytes"], 1000)++ def test_a_short_prompt_stays_on_the_decode_engine(self):+ self.run_one(list(range(50)))+ self.assertEqual(self.p.log, [])+ self.assertEqual(self.e.stats["short"], 1)++ def test_the_next_turn_is_not_handed_over_when_the_decode_engine_holds_it(self):+ ids = list(range(500))+ out = self.run_one(ids)+ turn2 = ids + out + list(range(7000, 7040)) # the answer and a short question+ self.run_one(turn2)+ self.assertEqual(len(self.p.log), 1) # only the first turn went through the prefill engine+ self.assertEqual(self.d.log[-1][0], "generate")+ self.assertGreater(self.d.log[-1][2], 490) # and the decode engine continued its own session++ def test_a_big_new_part_is_handed_over_again(self):+ ids = list(range(500))+ out = self.run_one(ids)+ self.run_one(ids + out + list(range(9000, 9400)))+ self.assertEqual(len(self.p.log), 2)++ def test_pictures_go_to_the_decode_engine(self):+ list(self.e.generate(list(range(500)), 3, {}, self.cancel, embeddings="/tmp/x"))+ self.assertEqual(self.p.log, [])++ def test_a_dead_prefill_engine_is_not_an_error(self):+ self.p.up = False+ self.e.heal = lambda: None+ out = self.run_one(list(range(500)))+ self.assertEqual(out, [100, 101, 102])+ self.assertEqual(self.d.log, [("generate", 500, 0)]) # the decode engine read all of it++ def test_a_prefill_engine_that_dies_midway_falls_back(self):+ self.p.fail_prefill = EngineDied("gone")+ self.e.heal = lambda: None+ out = self.run_one(list(range(500)))+ self.assertEqual(out, [100, 101, 102])+ self.assertEqual(self.d.log, [("generate", 500, 0)])+ self.assertTrue(any("failed" in s for s in self.said))++ def test_a_refused_file_falls_back(self):+ for which in ("fail_save", "fail_restore"):+ FakeEngine.files = {}+ self.p.session, self.d.session = [], []+ setattr(self.p if which == "fail_save" else self.d, which, SessionRefused("invalid", "other model", False))+ self.d.log.clear()+ ids = list(range(500)) if which == "fail_save" else list(range(1000, 1500))+ out = self.run_one(ids)+ self.assertEqual(out, [100, 101, 102])+ self.assertEqual(self.d.log, [("generate", 500, 0)], which)+ setattr(self.p if which == "fail_save" else self.d, which, None)+ self.assertEqual(self.e.stats["fallbacks"], 2)+ self.assertEqual(os.listdir(self.dir), [])++ def test_a_decode_engine_that_dies_in_the_restore_is_an_error(self):+ def boom(*a):+ self.d.up = False+ raise EngineDied("transfer failed")+ self.d.fail_restore = None+ orig = self.d.session_file+ self.d.session_file = lambda action, path: boom() if action == "restore" else orig(action, path)+ with self.assertRaises(EngineDied):+ self.run_one(list(range(500)))+ self.assertEqual(os.listdir(self.dir), [])++ def test_cancel_during_the_prefill_ends_quietly(self):+ self.cancel.set()+ orig = self.p.generate++ def gen(*a, **k):+ for t in orig(*a, **k):+ yield t+ self.p.generate = gen+ out = self.run_one(list(range(500)))+ self.assertEqual(out, [])+ self.assertEqual(self.d.log, [])+ self.assertEqual(self.e.held, [])++ def test_attributes_come_from_the_decode_engine(self):+ self.assertEqual(self.e.max_context, 4096)+ self.assertEqual(self.e.batch, 0)+ self.assertFalse(hasattr(self.e, "session_file"))+ self.assertTrue(self.e.alive())+ self.d.up = False+ self.assertFalse(self.e.alive())+ self.assertTrue(self.e.info["split_serve"])+ self.e.silence_s = 5+ self.assertEqual((self.p.silence_s, self.d.silence_s), (5, 5))++ def test_close_removes_the_work_dir(self):+ self.e.close()+ self.assertFalse(os.path.exists(self.dir))+ self.assertFalse(self.p.up or self.d.up)+++if __name__ == "__main__":+ unittest.main()diff --git a/src/program/generate.cpp b/src/program/generate.cppindex 4e871b0d2..a16001a2b 100644--- a/src/program/generate.cpp+++ b/src/program/generate.cpp@@ -12449,6 +12449,10 @@ int main(int argc, char** argv) {// parked after it. It still resumes from a checkpoint it matches, and still saves the system-prompt root// when that reaches --prompt-cache-root. Absent = checkpointed as before.int req_ckpt = 1;+ // prefill=1 (with max_new 0): split serving. Read the prompt as a request would, up to the place where the first+ // verify window starts (the session holds n - 1 tokens, exactly what a next turn that continues it needs), and+ // stop: no token is made. A SAVE after it is the hand-over to another engine (docs/SPLIT_SERVE.md).+ bool req_prefill_only = false;// pin=N: the first N prompt tokens are a shared read-only prefix (a long document that many short queries// follow). The prompt is read in two parts at N, the checkpoint there is PINNED (retention never evicts it,// a parked conversation holding it stays parked) and a later request that resumes from it does not park the@@ -12479,6 +12483,7 @@ int main(int argc, char** argv) {if (key == "logit_bias") { req_bias_text = tok.substr(eq + 1); req_bias_present = true; }else if (key == "cvec") req_cvec = std::atoi(tok.c_str() + eq + 1);else if (key == "ckpt") req_ckpt = std::atoi(tok.c_str() + eq + 1) != 0;+ else if (key == "prefill") req_prefill_only = std::atoi(tok.c_str() + eq + 1) != 0;else if (key == "pin") req_pin = std::max<long long>(0, std::atoll(tok.c_str() + eq + 1));else if (key == "temperature") req_temperature = fv;else if (key == "top_p") req_top_p = fv;@@ -12524,11 +12529,16 @@ int main(int argc, char** argv) {}std::vector<int64_t> ids;std::string pe;- if (max_new < 1 || endp == nullptr || (geni && emb_path.empty()) || !parse_i64_list(endp, ids, pe)) {+ if ((max_new < 1 && !(max_new == 0 && req_prefill_only)) || (req_prefill_only && max_new != 0) || endp == nullptr ||+ (geni && emb_path.empty()) || !parse_i64_list(endp, ids, pe)) {std::printf("ERR bad request: %s\n", pe.empty() ? "max_new" : pe.c_str());continue;}const int64_t n = (int64_t) ids.size();+ if (req_prefill_only && (geni || o.batch > 0 || n < 2)) {+ std::printf("ERR prefill=1 needs a text prompt of at least 2 tokens and no --batch\n");+ continue;+ }req_imgs.clear();if (geni && !o.vision) { std::printf("ERR this engine was started without --vision\n"); continue; }if (geni || !mrope_identity) {