Skip to content

The agent that grows with you

Merge remote-tracking branch 'origin/main' into feat/local-bolt-dflash

emozilla4c784612c028
diff --cc hermes_cli/local_runtime/bootstrap.py
index 725fba842e5,749e0e65f24..c7e27ff6989
--- a/hermes_cli/local_runtime/bootstrap.py
+++ b/hermes_cli/local_runtime/bootstrap.py
@@@ -230,7 -232,7 +232,8 @@@ def _admitted_models_max(mdir: Path, co
return cap
- def _generate_presets(mdir: Path, preset_path: Path, engine_tag: str | None = None) -> Path | None:
-def _generate_presets(mdir: Path, preset_path: Path, residents: int = 1) -> Path | None:
++def _generate_presets(mdir: Path, preset_path: Path, residents: int = 1, *,
++ engine_tag: str | None = None) -> Path | None:
"""Write the launch-policy INI for every staged model; returns the path to hand the router.
Priced against CAPACITY, not live free VRAM: this runs while the outgoing server instance may
@@@ -250,7 -254,7 +255,7 @@@
try:
capacity = probe_budget(planning=True)
for entry in generate_presets(mdir, capacity, preset_path, live=_launch_budget(capacity),
- engine_tag=engine_tag):
- residents=residents):
++ residents=residents, engine_tag=engine_tag):
if entry.refusal:
logger.warning("model refused by physics check: %s", entry.refusal)
return preset_path
@@@ -304,8 -308,7 +309,8 @@@ def refit_idle_presets(sup) -> bool
last = sup._refit_usable
if last is not None and abs(live.usable_vram_bytes - last) < _REFIT_STEP_BYTES:
return False
- text = render_presets(plan_presets(models_dir(), capacity, live=live,
- text = render_presets(plan_presets(models_dir(), capacity, live=live, residents=sup.models_max))
++ text = render_presets(plan_presets(models_dir(), capacity, live=live, residents=sup.models_max,
+ engine_tag=_SERVING_ENGINE.tag if _SERVING_ENGINE is not None else None))
# Held across the write and the reload so a restart can't spawn between them; everything
# slow ran above.
with sup._lifecycle_lock:
@@@ -455,11 -458,11 +460,12 @@@ def ensure_local_runtime(config: dict,
mdir = models_dir()
mdir.mkdir(parents=True, exist_ok=True)
- preset_path = _generate_presets(mdir, runtimes_root() / "presets.ini", engine.tag)
+ models_max = _admitted_models_max(mdir, int(section.get("models_max", 4)))
- preset_path = _generate_presets(mdir, runtimes_root() / "presets.ini", models_max)
++ preset_path = _generate_presets(mdir, runtimes_root() / "presets.ini", models_max,
++ engine_tag=engine.tag)
sup = LlamaServerSupervisor(engine.binary, mdir, preset_path=preset_path,
- models_max=_admitted_models_max(
- mdir, int(section.get("models_max", 4))),
+ models_max=models_max,
port=int(section.get("port", 0)) or None)
try:
sup.start()
diff --cc hermes_cli/local_runtime/presets.py
index 82764e54909,c0af3d860fe..d0164c41357
--- a/hermes_cli/local_runtime/presets.py
+++ b/hermes_cli/local_runtime/presets.py
@@@ -98,78 -98,14 +98,80 @@@ def _draft_fits(path: Path, profile, bu
and ctx_bytes(profile, window) + overhead + draft_need <= budget.usable_vram_bytes)
+# The draft model's own context beside the target's: its KV and compute buffers. Without it the
+# price below still covered every measured server (b11671, -ub 2048, one slot) by 0.2 GiB
+# (Nemotron 3.5 Bolt) to 2.7 GiB (Qwen3.8 27B); this keeps the thinnest case clear.
+_DRAFT_CONTEXT_BYTES = 512 << 20
+
+
+def _engine_build(tag: str | None) -> int | None:
+ try:
+ return int(str(tag).removeprefix("b")) if tag else None
+ except ValueError:
+ return None
+
+
+def _dflash_extra_bytes(entry, profile) -> int:
+ """Fixed bytes a one-slot DFlash draft adds beside its target: the draft's weights, a copy of
+ the recurrent state per draft position, and the draft's own context."""
+ copies = entry.draft_depth + 1
+ return entry.draft.size_bytes + profile.recurrent_state_bytes * copies + _DRAFT_CONTEXT_BYTES
+
+
+def _runs_dflash(entry) -> Path | None:
+ """The entry's DFlash draft on disk, or None."""
+ if entry is None or entry.draft is None or entry.draft_spec_type != "draft-dflash":
+ return None
+ return _asset_path(entry.draft)
+
+
+def _dflash_plan(entry, profile, budget: HardwareBudget, live: HardwareBudget | None, *,
+ fixed_overhead: int, requested_window: int | None, engine_tag: str | None,
+ window_without: int):
+ """The launch plan with the entry's DFlash draft in place of MTP, or None.
+
+ The draft runs on one slot: each slot keeps a recurrent-state copy per draft position, and
+ the router's default four slots hold enough copies to push weights off the GPU. It may
+ shrink the window to half native, never below what the model gets without it when that is
+ smaller; a requested (grown) window it can't hold keeps the model as it was.
+ """
+ if _runs_dflash(entry) is None:
+ return None
+ build = _engine_build(engine_tag)
+ if build is None or build < (_engine_build(entry.draft_min_engine) or 0):
+ return None
+ lean = replace(profile, kv_scale=1.0)
+ overhead = fixed_overhead + _dflash_extra_bytes(entry, lean)
+ plan = plan_launch(lean, budget, fixed_overhead=overhead, requested_window=requested_window)
+ if live is not None:
+ # fit_to_free_memory holds a too-busy card at the floor rather than move weights off it on a
+ # live reading. The draft never rides on that hold: it runs only where it fits now.
+ now = plan_launch(lean, live, fixed_overhead=overhead)
+ if isinstance(now.decision, PhysicsRefusal) or now.decision.spilled:
+ return None
+ plan = fit_to_free_memory(plan, lean, live, mtp_capable=False, fixed_overhead=overhead)
+ decision = plan.decision
+ if isinstance(decision, PhysicsRefusal) or decision.spilled:
+ return None
+ native = profile.n_ctx_train or decision.window
+ if requested_window and decision.window < min(requested_window, native):
+ return None
+ if decision.window < min(window_without, native // 2):
+ return None
+ return plan
+
+
def preset_for_model(gguf: Path, budget: HardwareBudget,
mtp_capable: set[str], *, requested_window: int | None = None,
- live: HardwareBudget | None = None,
- live: HardwareBudget | None = None, residents: int = 1) -> PresetEntry | None:
++ live: HardwareBudget | None = None, residents: int = 1,
+ engine_tag: str | None = None) -> PresetEntry | None:
"""The launch decision for one staged model, or None when its header is unreadable.
``budget`` is the card's capacity; ``live`` (from ``hardware.launch_budget``) narrows the
- window to what fits beside other programs now. ``engine_tag`` is the installed llama.cpp
- release; without it no draft that needs a minimum release runs.
+ window to what fits beside other programs now. ``residents`` is how many models the router may
+ hold at once; on Windows unified memory their prompt caches share the commit they leave.
++ ``engine_tag`` is the installed llama.cpp release; without it no draft that needs a minimum
++ release runs.
"""
from hermes_cli.local_runtime.catalog import entry_for_model
from hermes_cli.local_runtime.growth import load_window_overrides
@@@ -196,17 -132,16 +198,22 @@@
if isinstance(decision, PhysicsRefusal):
return PresetEntry(model_id=model_id, window=0, spilled=False, refusal=decision.message)
+ dflash = None if decision.spilled else _dflash_plan(
+ entry, profile, budget, live, fixed_overhead=fixed_overhead, requested_window=requested,
+ engine_tag=engine_tag, window_without=decision.window)
+ if dflash is not None:
+ plan, decision = dflash, dflash.decision
# Router discovery is preset-only: refused files must never autoload with stock fit.
keys = _args_to_keys(launch_args(
- profile, decision, mtp_capable=is_mtp, uma=budget.uma, mtp_prefill=plan.mtp_prefill,
+ profile, decision, mtp_capable=is_mtp and dflash is None, uma=budget.uma,
+ mtp_prefill=plan.mtp_prefill,
mtp_draft_depth=entry.mtp_draft_depth if entry is not None else 3))
keys["model"] = str(gguf)
+ from hermes_cli.local_runtime.hardware import prompt_cache_bytes
+ cache = prompt_cache_bytes(live, footprint_bytes(plan.profile, decision.window,
+ overhead_bytes=plan.overhead_bytes), residents)
+ if cache is not None:
+ keys["cache-ram"] = str(cache >> 20) # MiB; 0 turns the cache off
if profile.lazy_bytes:
# The price above leaves these tensors on disk; llama.cpp's own default loads them on
# integrated GPUs.
@@@ -293,13 -217,12 +300,14 @@@ def _launch_footprint(gguf: Path, budge
if isinstance(plan.decision, PhysicsRefusal):
return None
# Priced whole even when the plan spills: a spilled model still holds part of its weights on
- # the device, and over-counting errs toward the side that cannot thrash.
- return footprint_bytes(plan.profile, plan.decision.window, overhead_bytes=plan.overhead_bytes)
+ # the device, and over-counting errs toward the side that cannot thrash. A DFlash draft on
+ # disk counts whether or not this machine runs it, for the same reason.
+ draft = _dflash_extra_bytes(entry, profile) if _runs_dflash(entry) is not None else 0
+ return footprint_bytes(plan.profile, plan.decision.window, overhead_bytes=plan.overhead_bytes) + draft
- def admitted_residency_count(models_dir: Path, budget: HardwareBudget, configured: int) -> int:
+ def admitted_residency_count(models_dir: Path, budget: HardwareBudget, configured: int,
+ live: HardwareBudget | None = None) -> int:
"""How many models the card may hold resident at once: priced against the budget, not a count.
Residency used to be bounded by a count alone, so a second model was admitted against an
@@@ -329,23 -256,21 +341,25 @@@
def plan_presets(models_dir: Path, budget: HardwareBudget, mtp_capable: set[str] | None = None,
- *, live: HardwareBudget | None = None,
- *, live: HardwareBudget | None = None, residents: int = 1) -> list[PresetEntry]:
++ *, live: HardwareBudget | None = None, residents: int = 1,
+ engine_tag: str | None = None) -> list[PresetEntry]:
"""The launch decision for every staged model; unreadable headers are skipped."""
from hermes_cli.local_runtime.bootstrap import staged_in
- entries = (preset_for_model(gguf, budget, mtp_capable or set(), live=live, engine_tag=engine_tag)
- entries = (preset_for_model(gguf, budget, mtp_capable or set(), live=live, residents=residents)
++ entries = (preset_for_model(gguf, budget, mtp_capable or set(), live=live, residents=residents,
++ engine_tag=engine_tag)
for gguf in staged_in(models_dir))
return [entry for entry in entries if entry is not None]
def generate_presets(models_dir: Path, budget: HardwareBudget, preset_path: Path,
mtp_capable: set[str] | None = None, *,
- live: HardwareBudget | None = None,
- live: HardwareBudget | None = None, residents: int = 1) -> list[PresetEntry]:
++ live: HardwareBudget | None = None, residents: int = 1,
+ engine_tag: str | None = None) -> list[PresetEntry]:
"""Plan every staged model and write one INI. Refused models get no section (the picker
surfaces the refusal from the returned entries)."""
- entries = plan_presets(models_dir, budget, mtp_capable, live=live, engine_tag=engine_tag)
- entries = plan_presets(models_dir, budget, mtp_capable, live=live, residents=residents)
++ entries = plan_presets(models_dir, budget, mtp_capable, live=live, residents=residents,
++ engine_tag=engine_tag)
write_presets(entries, preset_path)
return entries
diff --cc tests/hermes_cli/test_local_launch_budget.py
index 42640769af9,355d2efbbe1..121e158a4fc
--- a/tests/hermes_cli/test_local_launch_budget.py
+++ b/tests/hermes_cli/test_local_launch_budget.py
@@@ -261,28 -263,6 +263,28 @@@ def test_idle_router_gets_windows_for_t
assert bootstrap.refit_idle_presets(router) is False and router.reloads == 1
+def test_a_replan_keeps_the_draft_the_serving_engine_loads(hermes_home, monkeypatch, capacity_probe):
+ """The idle re-plan decides for the engine the server runs, as boot did: a staged DFlash draft
+ stays in the preset, not the MTP fallback an unknown engine gets."""
+ from hermes_cli.local_runtime import bootstrap, catalog
+
+ mdir, _gguf = _stage_27b(hermes_home, monkeypatch)
+ draft = catalog.catalog_by_id()["qwen3.8-27b"].draft
+ (bootstrap.assets_dir() / draft.local_name).touch()
+ engine = SimpleNamespace(tag="b11671")
+ monkeypatch.setattr(bootstrap, "_SERVING_ENGINE", engine)
+ _card(monkeypatch, 2.0)
+ path = hermes_home / "presets.ini"
- bootstrap._generate_presets(mdir, path, engine.tag)
++ bootstrap._generate_presets(mdir, path, engine_tag=engine.tag)
+ assert "spec-type = draft-dflash" in path.read_text(encoding="utf-8")
+
+ _card(monkeypatch, 0.0) # the user closed the heavy apps
+ bootstrap.refit_idle_presets(_Router(path, {MODEL_ID: "unloaded"}))
+
+ text = path.read_text(encoding="utf-8")
+ assert "spec-type = draft-dflash" in text and "parallel = 1" in text
+
+
@pytest.mark.parametrize("status", ["loaded", "loading", "sleeping"])
def test_never_rewrites_while_a_model_holds_memory(hermes_home, monkeypatch, capacity_probe, status):
"""The router unloads a loaded model whose launch flags change on reload."""
diff --cc tests/hermes_cli/test_local_residency_admission.py
index 41b6e8c43cb,b98208071ea..d9e973a37f0
--- a/tests/hermes_cli/test_local_residency_admission.py
+++ b/tests/hermes_cli/test_local_residency_admission.py
@@@ -83,7 -83,7 +83,7 @@@ def test_boot_hands_the_router_the_deri
monkeypatch.setattr(bs, "models_dir", lambda: mdir)
monkeypatch.setattr(hardware, "probe_budget", probe)
monkeypatch.setattr(bs, "_SUPERVISOR", None)
- monkeypatch.setattr(bs, "_generate_presets", lambda mdir, preset_path, engine_tag=None: None)
- monkeypatch.setattr(bs, "_generate_presets", lambda mdir, preset_path, residents=1: None)
++ monkeypatch.setattr(bs, "_generate_presets", lambda mdir, preset_path, residents=1, engine_tag=None: None)
monkeypatch.setattr(bs, "_detect_gpu_vendor", lambda: None)
monkeypatch.setattr(binaries, "installed_engine",
lambda backend="auto", **k: binaries.Engine("cpu", "build-1", tmp_path / "install" / "llama-server"))
diff --cc tests/hermes_cli/test_local_runtime.py
index d9c7af97347,11094e12e51..b9ef4cf13ca
--- a/tests/hermes_cli/test_local_runtime.py
+++ b/tests/hermes_cli/test_local_runtime.py
@@@ -1004,8 -1004,8 +1004,12 @@@ def test_boot_prices_the_engine_it_serv
monkeypatch.setattr("hermes_cli.config.load_config_readonly",
lambda: {"local_runtime": {"backend": "cpu"}}) # another profile's config
priced: list = []
- monkeypatch.setattr(bootstrap, "_generate_presets", lambda mdir, path, engine_tag=None: priced.append(
- (hardware._configured_engine(), engine_tag)) or path)
- monkeypatch.setattr(bootstrap, "_generate_presets",
- lambda mdir, path, residents=1: priced.append(hardware._configured_engine()) or path)
++
++ def _priced(mdir, path, residents=1, engine_tag=None):
++ priced.append((hardware._configured_engine(), engine_tag))
++ return path
++
++ monkeypatch.setattr(bootstrap, "_generate_presets", _priced)
monkeypatch.setattr(bootstrap, "_admitted_models_max", lambda mdir, configured: configured)
class _NoStart:
diff --cc tests/hermes_cli/test_local_runtime_updates.py
index ffa1c10f0f1,ffa1c10f0f1..dbb1b5b3dd5
--- a/tests/hermes_cli/test_local_runtime_updates.py
+++ b/tests/hermes_cli/test_local_runtime_updates.py
@@@ -64,7 -64,7 +64,7 @@@ def test_boot_uses_previous_pm_engine_w
binary = record_engine(store, "10362")
monkeypatch.setattr(bootstrap, "_SUPERVISOR", None)
monkeypatch.setattr(endpoint, "_state_endpoint", lambda: None)
-- monkeypatch.setattr(bootstrap, "_generate_presets", lambda *args: None)
++ monkeypatch.setattr(bootstrap, "_generate_presets", lambda *args, **kwargs: None)
monkeypatch.setattr(bootstrap, "_start_idle_sweeper", lambda *args: None)
def forbidden(*args, **kwargs):