The agent that grows with you
Python aiai-agentai-agentsanthropicchatgptclaudeclaude-codecodexhermeshermes-agentllmnous-researchopenai
Merge remote-tracking branch 'origin/main' into feat/local-bolt-dflash
emozilla4c784612c028
diff --cc hermes_cli/local_runtime/bootstrap.pyindex 725fba842e5,749e0e65f24..c7e27ff6989--- a/hermes_cli/local_runtime/bootstrap.py+++ b/hermes_cli/local_runtime/bootstrap.py@@@ -230,7 -232,7 +232,8 @@@ def _admitted_models_max(mdir: Path, coreturn cap- def _generate_presets(mdir: Path, preset_path: Path, engine_tag: str | None = None) -> Path | None:-def _generate_presets(mdir: Path, preset_path: Path, residents: int = 1) -> Path | None:++def _generate_presets(mdir: Path, preset_path: Path, residents: int = 1, *,++ engine_tag: str | None = None) -> Path | None:"""Write the launch-policy INI for every staged model; returns the path to hand the router.Priced against CAPACITY, not live free VRAM: this runs while the outgoing server instance may@@@ -250,7 -254,7 +255,7 @@@try:capacity = probe_budget(planning=True)for entry in generate_presets(mdir, capacity, preset_path, live=_launch_budget(capacity),- engine_tag=engine_tag):- residents=residents):++ residents=residents, engine_tag=engine_tag):if entry.refusal:logger.warning("model refused by physics check: %s", entry.refusal)return preset_path@@@ -304,8 -308,7 +309,8 @@@ def refit_idle_presets(sup) -> boollast = sup._refit_usableif last is not None and abs(live.usable_vram_bytes - last) < _REFIT_STEP_BYTES:return False- text = render_presets(plan_presets(models_dir(), capacity, live=live,- text = render_presets(plan_presets(models_dir(), capacity, live=live, residents=sup.models_max))++ text = render_presets(plan_presets(models_dir(), capacity, live=live, residents=sup.models_max,+ engine_tag=_SERVING_ENGINE.tag if _SERVING_ENGINE is not None else None))# Held across the write and the reload so a restart can't spawn between them; everything# slow ran above.with sup._lifecycle_lock:@@@ -455,11 -458,11 +460,12 @@@ def ensure_local_runtime(config: dict,mdir = models_dir()mdir.mkdir(parents=True, exist_ok=True)- preset_path = _generate_presets(mdir, runtimes_root() / "presets.ini", engine.tag)+ models_max = _admitted_models_max(mdir, int(section.get("models_max", 4)))- preset_path = _generate_presets(mdir, runtimes_root() / "presets.ini", models_max)++ preset_path = _generate_presets(mdir, runtimes_root() / "presets.ini", models_max,++ engine_tag=engine.tag)sup = LlamaServerSupervisor(engine.binary, mdir, preset_path=preset_path,- models_max=_admitted_models_max(- mdir, int(section.get("models_max", 4))),+ models_max=models_max,port=int(section.get("port", 0)) or None)try:sup.start()diff --cc hermes_cli/local_runtime/presets.pyindex 82764e54909,c0af3d860fe..d0164c41357--- a/hermes_cli/local_runtime/presets.py+++ b/hermes_cli/local_runtime/presets.py@@@ -98,78 -98,14 +98,80 @@@ def _draft_fits(path: Path, profile, buand ctx_bytes(profile, window) + overhead + draft_need <= budget.usable_vram_bytes)+# The draft model's own context beside the target's: its KV and compute buffers. Without it the+# price below still covered every measured server (b11671, -ub 2048, one slot) by 0.2 GiB+# (Nemotron 3.5 Bolt) to 2.7 GiB (Qwen3.8 27B); this keeps the thinnest case clear.+_DRAFT_CONTEXT_BYTES = 512 << 20+++def _engine_build(tag: str | None) -> int | None:+ try:+ return int(str(tag).removeprefix("b")) if tag else None+ except ValueError:+ return None+++def _dflash_extra_bytes(entry, profile) -> int:+ """Fixed bytes a one-slot DFlash draft adds beside its target: the draft's weights, a copy of+ the recurrent state per draft position, and the draft's own context."""+ copies = entry.draft_depth + 1+ return entry.draft.size_bytes + profile.recurrent_state_bytes * copies + _DRAFT_CONTEXT_BYTES+++def _runs_dflash(entry) -> Path | None:+ """The entry's DFlash draft on disk, or None."""+ if entry is None or entry.draft is None or entry.draft_spec_type != "draft-dflash":+ return None+ return _asset_path(entry.draft)+++def _dflash_plan(entry, profile, budget: HardwareBudget, live: HardwareBudget | None, *,+ fixed_overhead: int, requested_window: int | None, engine_tag: str | None,+ window_without: int):+ """The launch plan with the entry's DFlash draft in place of MTP, or None.++ The draft runs on one slot: each slot keeps a recurrent-state copy per draft position, and+ the router's default four slots hold enough copies to push weights off the GPU. It may+ shrink the window to half native, never below what the model gets without it when that is+ smaller; a requested (grown) window it can't hold keeps the model as it was.+ """+ if _runs_dflash(entry) is None:+ return None+ build = _engine_build(engine_tag)+ if build is None or build < (_engine_build(entry.draft_min_engine) or 0):+ return None+ lean = replace(profile, kv_scale=1.0)+ overhead = fixed_overhead + _dflash_extra_bytes(entry, lean)+ plan = plan_launch(lean, budget, fixed_overhead=overhead, requested_window=requested_window)+ if live is not None:+ # fit_to_free_memory holds a too-busy card at the floor rather than move weights off it on a+ # live reading. The draft never rides on that hold: it runs only where it fits now.+ now = plan_launch(lean, live, fixed_overhead=overhead)+ if isinstance(now.decision, PhysicsRefusal) or now.decision.spilled:+ return None+ plan = fit_to_free_memory(plan, lean, live, mtp_capable=False, fixed_overhead=overhead)+ decision = plan.decision+ if isinstance(decision, PhysicsRefusal) or decision.spilled:+ return None+ native = profile.n_ctx_train or decision.window+ if requested_window and decision.window < min(requested_window, native):+ return None+ if decision.window < min(window_without, native // 2):+ return None+ return plan++def preset_for_model(gguf: Path, budget: HardwareBudget,mtp_capable: set[str], *, requested_window: int | None = None,- live: HardwareBudget | None = None,- live: HardwareBudget | None = None, residents: int = 1) -> PresetEntry | None:++ live: HardwareBudget | None = None, residents: int = 1,+ engine_tag: str | None = None) -> PresetEntry | None:"""The launch decision for one staged model, or None when its header is unreadable.``budget`` is the card's capacity; ``live`` (from ``hardware.launch_budget``) narrows the- window to what fits beside other programs now. ``engine_tag`` is the installed llama.cpp- release; without it no draft that needs a minimum release runs.+ window to what fits beside other programs now. ``residents`` is how many models the router may+ hold at once; on Windows unified memory their prompt caches share the commit they leave.++ ``engine_tag`` is the installed llama.cpp release; without it no draft that needs a minimum++ release runs."""from hermes_cli.local_runtime.catalog import entry_for_modelfrom hermes_cli.local_runtime.growth import load_window_overrides@@@ -196,17 -132,16 +198,22 @@@if isinstance(decision, PhysicsRefusal):return PresetEntry(model_id=model_id, window=0, spilled=False, refusal=decision.message)+ dflash = None if decision.spilled else _dflash_plan(+ entry, profile, budget, live, fixed_overhead=fixed_overhead, requested_window=requested,+ engine_tag=engine_tag, window_without=decision.window)+ if dflash is not None:+ plan, decision = dflash, dflash.decision# Router discovery is preset-only: refused files must never autoload with stock fit.keys = _args_to_keys(launch_args(- profile, decision, mtp_capable=is_mtp, uma=budget.uma, mtp_prefill=plan.mtp_prefill,+ profile, decision, mtp_capable=is_mtp and dflash is None, uma=budget.uma,+ mtp_prefill=plan.mtp_prefill,mtp_draft_depth=entry.mtp_draft_depth if entry is not None else 3))keys["model"] = str(gguf)+ from hermes_cli.local_runtime.hardware import prompt_cache_bytes+ cache = prompt_cache_bytes(live, footprint_bytes(plan.profile, decision.window,+ overhead_bytes=plan.overhead_bytes), residents)+ if cache is not None:+ keys["cache-ram"] = str(cache >> 20) # MiB; 0 turns the cache offif profile.lazy_bytes:# The price above leaves these tensors on disk; llama.cpp's own default loads them on# integrated GPUs.@@@ -293,13 -217,12 +300,14 @@@ def _launch_footprint(gguf: Path, budgeif isinstance(plan.decision, PhysicsRefusal):return None# Priced whole even when the plan spills: a spilled model still holds part of its weights on- # the device, and over-counting errs toward the side that cannot thrash.- return footprint_bytes(plan.profile, plan.decision.window, overhead_bytes=plan.overhead_bytes)+ # the device, and over-counting errs toward the side that cannot thrash. A DFlash draft on+ # disk counts whether or not this machine runs it, for the same reason.+ draft = _dflash_extra_bytes(entry, profile) if _runs_dflash(entry) is not None else 0+ return footprint_bytes(plan.profile, plan.decision.window, overhead_bytes=plan.overhead_bytes) + draft- def admitted_residency_count(models_dir: Path, budget: HardwareBudget, configured: int) -> int:+ def admitted_residency_count(models_dir: Path, budget: HardwareBudget, configured: int,+ live: HardwareBudget | None = None) -> int:"""How many models the card may hold resident at once: priced against the budget, not a count.Residency used to be bounded by a count alone, so a second model was admitted against an@@@ -329,23 -256,21 +341,25 @@@def plan_presets(models_dir: Path, budget: HardwareBudget, mtp_capable: set[str] | None = None,- *, live: HardwareBudget | None = None,- *, live: HardwareBudget | None = None, residents: int = 1) -> list[PresetEntry]:++ *, live: HardwareBudget | None = None, residents: int = 1,+ engine_tag: str | None = None) -> list[PresetEntry]:"""The launch decision for every staged model; unreadable headers are skipped."""from hermes_cli.local_runtime.bootstrap import staged_in- entries = (preset_for_model(gguf, budget, mtp_capable or set(), live=live, engine_tag=engine_tag)- entries = (preset_for_model(gguf, budget, mtp_capable or set(), live=live, residents=residents)++ entries = (preset_for_model(gguf, budget, mtp_capable or set(), live=live, residents=residents,++ engine_tag=engine_tag)for gguf in staged_in(models_dir))return [entry for entry in entries if entry is not None]def generate_presets(models_dir: Path, budget: HardwareBudget, preset_path: Path,mtp_capable: set[str] | None = None, *,- live: HardwareBudget | None = None,- live: HardwareBudget | None = None, residents: int = 1) -> list[PresetEntry]:++ live: HardwareBudget | None = None, residents: int = 1,+ engine_tag: str | None = None) -> list[PresetEntry]:"""Plan every staged model and write one INI. Refused models get no section (the pickersurfaces the refusal from the returned entries)."""- entries = plan_presets(models_dir, budget, mtp_capable, live=live, engine_tag=engine_tag)- entries = plan_presets(models_dir, budget, mtp_capable, live=live, residents=residents)++ entries = plan_presets(models_dir, budget, mtp_capable, live=live, residents=residents,++ engine_tag=engine_tag)write_presets(entries, preset_path)return entriesdiff --cc tests/hermes_cli/test_local_launch_budget.pyindex 42640769af9,355d2efbbe1..121e158a4fc--- a/tests/hermes_cli/test_local_launch_budget.py+++ b/tests/hermes_cli/test_local_launch_budget.py@@@ -261,28 -263,6 +263,28 @@@ def test_idle_router_gets_windows_for_tassert bootstrap.refit_idle_presets(router) is False and router.reloads == 1+def test_a_replan_keeps_the_draft_the_serving_engine_loads(hermes_home, monkeypatch, capacity_probe):+ """The idle re-plan decides for the engine the server runs, as boot did: a staged DFlash draft+ stays in the preset, not the MTP fallback an unknown engine gets."""+ from hermes_cli.local_runtime import bootstrap, catalog++ mdir, _gguf = _stage_27b(hermes_home, monkeypatch)+ draft = catalog.catalog_by_id()["qwen3.8-27b"].draft+ (bootstrap.assets_dir() / draft.local_name).touch()+ engine = SimpleNamespace(tag="b11671")+ monkeypatch.setattr(bootstrap, "_SERVING_ENGINE", engine)+ _card(monkeypatch, 2.0)+ path = hermes_home / "presets.ini"- bootstrap._generate_presets(mdir, path, engine.tag)++ bootstrap._generate_presets(mdir, path, engine_tag=engine.tag)+ assert "spec-type = draft-dflash" in path.read_text(encoding="utf-8")++ _card(monkeypatch, 0.0) # the user closed the heavy apps+ bootstrap.refit_idle_presets(_Router(path, {MODEL_ID: "unloaded"}))++ text = path.read_text(encoding="utf-8")+ assert "spec-type = draft-dflash" in text and "parallel = 1" in text++@pytest.mark.parametrize("status", ["loaded", "loading", "sleeping"])def test_never_rewrites_while_a_model_holds_memory(hermes_home, monkeypatch, capacity_probe, status):"""The router unloads a loaded model whose launch flags change on reload."""diff --cc tests/hermes_cli/test_local_residency_admission.pyindex 41b6e8c43cb,b98208071ea..d9e973a37f0--- a/tests/hermes_cli/test_local_residency_admission.py+++ b/tests/hermes_cli/test_local_residency_admission.py@@@ -83,7 -83,7 +83,7 @@@ def test_boot_hands_the_router_the_derimonkeypatch.setattr(bs, "models_dir", lambda: mdir)monkeypatch.setattr(hardware, "probe_budget", probe)monkeypatch.setattr(bs, "_SUPERVISOR", None)- monkeypatch.setattr(bs, "_generate_presets", lambda mdir, preset_path, engine_tag=None: None)- monkeypatch.setattr(bs, "_generate_presets", lambda mdir, preset_path, residents=1: None)++ monkeypatch.setattr(bs, "_generate_presets", lambda mdir, preset_path, residents=1, engine_tag=None: None)monkeypatch.setattr(bs, "_detect_gpu_vendor", lambda: None)monkeypatch.setattr(binaries, "installed_engine",lambda backend="auto", **k: binaries.Engine("cpu", "build-1", tmp_path / "install" / "llama-server"))diff --cc tests/hermes_cli/test_local_runtime.pyindex d9c7af97347,11094e12e51..b9ef4cf13ca--- a/tests/hermes_cli/test_local_runtime.py+++ b/tests/hermes_cli/test_local_runtime.py@@@ -1004,8 -1004,8 +1004,12 @@@ def test_boot_prices_the_engine_it_servmonkeypatch.setattr("hermes_cli.config.load_config_readonly",lambda: {"local_runtime": {"backend": "cpu"}}) # another profile's configpriced: list = []- monkeypatch.setattr(bootstrap, "_generate_presets", lambda mdir, path, engine_tag=None: priced.append(- (hardware._configured_engine(), engine_tag)) or path)- monkeypatch.setattr(bootstrap, "_generate_presets",- lambda mdir, path, residents=1: priced.append(hardware._configured_engine()) or path)++++ def _priced(mdir, path, residents=1, engine_tag=None):++ priced.append((hardware._configured_engine(), engine_tag))++ return path++++ monkeypatch.setattr(bootstrap, "_generate_presets", _priced)monkeypatch.setattr(bootstrap, "_admitted_models_max", lambda mdir, configured: configured)class _NoStart:diff --cc tests/hermes_cli/test_local_runtime_updates.pyindex ffa1c10f0f1,ffa1c10f0f1..dbb1b5b3dd5--- a/tests/hermes_cli/test_local_runtime_updates.py+++ b/tests/hermes_cli/test_local_runtime_updates.py@@@ -64,7 -64,7 +64,7 @@@ def test_boot_uses_previous_pm_engine_wbinary = record_engine(store, "10362")monkeypatch.setattr(bootstrap, "_SUPERVISOR", None)monkeypatch.setattr(endpoint, "_state_endpoint", lambda: None)-- monkeypatch.setattr(bootstrap, "_generate_presets", lambda *args: None)++ monkeypatch.setattr(bootstrap, "_generate_presets", lambda *args, **kwargs: None)monkeypatch.setattr(bootstrap, "_start_idle_sweeper", lambda *args: None)def forbidden(*args, **kwargs):