The agent that grows with you
Python aiai-agentai-agentsanthropicchatgptclaudeclaude-codecodexhermeshermes-agentllmnous-researchopenai
fix(local-runtime): price a drafted model once in the residency cap
emozilla6e32bb101056
diff --git a/hermes_cli/local_runtime/bootstrap.py b/hermes_cli/local_runtime/bootstrap.pyindex c7e27ff6989..b23950044be 100644--- a/hermes_cli/local_runtime/bootstrap.py+++ b/hermes_cli/local_runtime/bootstrap.py@@ -208,7 +208,7 @@ def refresh_local_runtime() -> bool:return False-def _admitted_models_max(mdir: Path, configured: int) -> int:+def _admitted_models_max(mdir: Path, configured: int, engine_tag: str | None = None) -> int:"""Residency cap to hand the router: derived from the hardware budget, ``models_max`` as a ceiling.A cap of "four" on a card that holds one model is how a second child ends up paged (WDDM) and@@ -222,7 +222,7 @@ def _admitted_models_max(mdir: Path, configured: int) -> int:capacity = probe_budget(planning=True)live = _launch_budget(capacity) if capacity.uma else None- cap = admitted_residency_count(mdir, capacity, configured, live=live)+ cap = admitted_residency_count(mdir, capacity, configured, live=live, engine_tag=engine_tag)except Exception as exc:logger.warning("residency cap probe failed (%s); using models_max=%s", exc, configured)return configured@@ -460,7 +460,7 @@ def ensure_local_runtime(config: dict, force: bool = False) -> object | None:mdir = models_dir()mdir.mkdir(parents=True, exist_ok=True)- models_max = _admitted_models_max(mdir, int(section.get("models_max", 4)))+ models_max = _admitted_models_max(mdir, int(section.get("models_max", 4)), engine.tag)preset_path = _generate_presets(mdir, runtimes_root() / "presets.ini", models_max,engine_tag=engine.tag)diff --git a/hermes_cli/local_runtime/presets.py b/hermes_cli/local_runtime/presets.pyindex 065e08a3cee..62e966ce5a8 100644--- a/hermes_cli/local_runtime/presets.py+++ b/hermes_cli/local_runtime/presets.py@@ -285,7 +285,7 @@ def resident_footprint(gguf: Path, budget: HardwareBudget, window: int, *, draftreturn footprint_bytes(plan.profile, window, overhead_bytes=plan.overhead_bytes)-def _launch_footprint(gguf: Path, budget: HardwareBudget) -> int | None:+def _launch_footprint(gguf: Path, budget: HardwareBudget, engine_tag: str | None = None) -> int | None:"""Estimated resident bytes for one staged model at the window this policy grants it, or Nonewhen it cannot be priced: an unreadable header, or a model the physics check refuses outright(it never loads, so it must not shrink the residency cap)."""@@ -300,20 +300,28 @@ def _launch_footprint(gguf: Path, budget: HardwareBudget) -> int | None:return Noneentry = entry_for_model(model_id)companions = _companions(entry)+ fixed_overhead = RUNTIME_OVERHEAD_BYTES + companions.nbytes+ requested = load_window_overrides().get(model_id)plan = plan_launch(profile, budget, mtp_capable=_runs_mtp(entry, companions),- fixed_overhead=RUNTIME_OVERHEAD_BYTES + companions.nbytes,- requested_window=load_window_overrides().get(model_id))+ fixed_overhead=fixed_overhead, requested_window=requested)if isinstance(plan.decision, PhysicsRefusal):return None# Priced whole even when the plan spills: a spilled model still holds part of its weights on- # the device, and over-counting errs toward the side that cannot thrash. A DFlash draft on- # disk counts whether or not this machine runs it, for the same reason.- draft = _dflash_extra_bytes(entry, profile) if _runs_dflash(entry) is not None else 0- return footprint_bytes(plan.profile, plan.decision.window, overhead_bytes=plan.overhead_bytes) + draft+ # the device, and over-counting errs toward the side that cannot thrash.+ price = footprint_bytes(plan.profile, plan.decision.window, overhead_bytes=plan.overhead_bytes)+ # Where the engine loads the draft, the preset runs it in place of MTP: the model holds the+ # larger of the two launches, never both.+ dflash = None if plan.decision.spilled else _dflash_plan(+ entry, profile, budget, None, fixed_overhead=fixed_overhead, requested_window=requested,+ engine_tag=engine_tag, window_without=plan.decision.window)+ if dflash is not None:+ price = max(price, footprint_bytes(dflash.profile, dflash.decision.window,+ overhead_bytes=dflash.overhead_bytes))+ return pricedef admitted_residency_count(models_dir: Path, budget: HardwareBudget, configured: int,- live: HardwareBudget | None = None) -> int:+ live: HardwareBudget | None = None, engine_tag: str | None = None) -> int:"""How many models the card may hold resident at once: priced against the budget, not a count.Residency used to be bounded by a count alone, so a second model was admitted against an@@ -335,7 +343,7 @@ def admitted_residency_count(models_dir: Path, budget: HardwareBudget, configurereturn configuredlargest = 0for gguf in staged_in(models_dir):- need = _launch_footprint(gguf, budget)+ need = _launch_footprint(gguf, budget, engine_tag)if need:largest = max(largest, need)if largest <= 0:diff --git a/tests/hermes_cli/test_local_external_draft.py b/tests/hermes_cli/test_local_external_draft.pyindex ff2510db7f5..c6a9263fc45 100644--- a/tests/hermes_cli/test_local_external_draft.py+++ b/tests/hermes_cli/test_local_external_draft.py@@ -221,12 +221,31 @@ def test_a_model_without_mtp_drafts_without_backend_sampling(model):assert "backend-sampling" not in preset.keys-def test_the_residency_price_includes_the_draft_once_it_is_on_disk(model):- gguf, draft, _ = model- with_draft = presets._launch_footprint(gguf, ROOMY)- draft.unlink()+def test_the_residency_price_is_the_launch_the_model_runs(model):+ """The preset runs the draft in place of MTP, so the model holds the larger of the two launches,+ never both added together. An engine that can't load the draft prices MTP alone."""+ gguf, _, use = model+ larger = set()+ for state in (150 * MIB, 600 * MIB): # MTP's own layers outweigh the draft, then the reverse+ use(state=state)+ mtp = presets.resident_footprint(gguf, ROOMY, NATIVE)+ drafting = presets.resident_footprint(gguf, ROOMY, NATIVE, drafting=True)+ larger.add("draft" if drafting > mtp else "mtp")++ assert presets._launch_footprint(gguf, ROOMY, engine_tag="b11671") == max(mtp, drafting)+ assert presets._launch_footprint(gguf, ROOMY, engine_tag="b11000") == mtp+ assert presets._launch_footprint(gguf, ROOMY) == mtp+ assert larger == {"draft", "mtp"}+++def test_two_models_stay_resident_where_two_drafted_launches_fit(model):+ gguf, _, _ = model+ gguf.touch()+ held = max(presets.resident_footprint(gguf, ROOMY, NATIVE),+ presets.resident_footprint(gguf, ROOMY, NATIVE, drafting=True))+ card = replace(ROOMY, usable_vram_bytes=2 * held + GIB)- assert with_draft >= presets._launch_footprint(gguf, ROOMY) + ENTRY.draft.size_bytes+ assert presets.admitted_residency_count(gguf.parent, card, 4, engine_tag="b11671") == 2def _staged(monkeypatch, *entries):diff --git a/tests/hermes_cli/test_local_runtime.py b/tests/hermes_cli/test_local_runtime.pyindex b9ef4cf13ca..4ccb8ad7be7 100644--- a/tests/hermes_cli/test_local_runtime.py+++ b/tests/hermes_cli/test_local_runtime.py@@ -1010,7 +1010,9 @@ def test_boot_prices_the_engine_it_serves(tmp_path, monkeypatch):return pathmonkeypatch.setattr(bootstrap, "_generate_presets", _priced)- monkeypatch.setattr(bootstrap, "_admitted_models_max", lambda mdir, configured: configured)+ capped: list = []+ monkeypatch.setattr(bootstrap, "_admitted_models_max",+ lambda mdir, configured, engine_tag=None: capped.append(engine_tag) or configured)class _NoStart:def __init__(self, *a, **k):@@ -1027,4 +1029,5 @@ def test_boot_prices_the_engine_it_serves(tmp_path, monkeypatch):assert bootstrap.ensure_local_runtime({"local_runtime": {"enabled": True, "backend": "vulkan"}}) is None# The release reaches the presets too: drafts gate on the engine that will load them.assert priced == [(vulkan, "b1")]+ assert capped == ["b1"] # the residency cap prices the draft only where this engine loads itassert bootstrap.serving_engine() is None # a failed boot serves nothingdiff --git a/tests/hermes_cli/test_local_windows_commit_budget.py b/tests/hermes_cli/test_local_windows_commit_budget.pyindex feab6721b95..d47b4fa5c1f 100644--- a/tests/hermes_cli/test_local_windows_commit_budget.py+++ b/tests/hermes_cli/test_local_windows_commit_budget.py@@ -103,7 +103,7 @@ def test_router_holds_only_the_models_free_commit_fits(tmp_path, monkeypatch):from hermes_cli.local_runtime import bootstrapmonkeypatch.setattr(bootstrap, "staged_in", lambda mdir: [tmp_path / "a.gguf", tmp_path / "b.gguf"])- monkeypatch.setattr(presets, "_launch_footprint", lambda gguf, budget: 12 * G)+ monkeypatch.setattr(presets, "_launch_footprint", lambda gguf, budget, engine_tag=None: 12 * G)capacity = HardwareBudget(int(30 * G), int(50 * G), 0, uma=True)live = HardwareBudget(int(16 * G), int(50 * G), 0, uma=True, commit_room_bytes=int(20 * G))assert presets.admitted_residency_count(tmp_path, capacity, 4) == 2