Skip to content

The agent that grows with you

fix(local-runtime): price a drafted model once in the residency cap

emozilla6e32bb101056
diff --git a/hermes_cli/local_runtime/bootstrap.py b/hermes_cli/local_runtime/bootstrap.py
index c7e27ff6989..b23950044be 100644
--- a/hermes_cli/local_runtime/bootstrap.py
+++ b/hermes_cli/local_runtime/bootstrap.py
@@ -208,7 +208,7 @@ def refresh_local_runtime() -> bool:
return False
-def _admitted_models_max(mdir: Path, configured: int) -> int:
+def _admitted_models_max(mdir: Path, configured: int, engine_tag: str | None = None) -> int:
"""Residency cap to hand the router: derived from the hardware budget, ``models_max`` as a ceiling.
A cap of "four" on a card that holds one model is how a second child ends up paged (WDDM) and
@@ -222,7 +222,7 @@ def _admitted_models_max(mdir: Path, configured: int) -> int:
capacity = probe_budget(planning=True)
live = _launch_budget(capacity) if capacity.uma else None
- cap = admitted_residency_count(mdir, capacity, configured, live=live)
+ cap = admitted_residency_count(mdir, capacity, configured, live=live, engine_tag=engine_tag)
except Exception as exc:
logger.warning("residency cap probe failed (%s); using models_max=%s", exc, configured)
return configured
@@ -460,7 +460,7 @@ def ensure_local_runtime(config: dict, force: bool = False) -> object | None:
mdir = models_dir()
mdir.mkdir(parents=True, exist_ok=True)
- models_max = _admitted_models_max(mdir, int(section.get("models_max", 4)))
+ models_max = _admitted_models_max(mdir, int(section.get("models_max", 4)), engine.tag)
preset_path = _generate_presets(mdir, runtimes_root() / "presets.ini", models_max,
engine_tag=engine.tag)
diff --git a/hermes_cli/local_runtime/presets.py b/hermes_cli/local_runtime/presets.py
index 065e08a3cee..62e966ce5a8 100644
--- a/hermes_cli/local_runtime/presets.py
+++ b/hermes_cli/local_runtime/presets.py
@@ -285,7 +285,7 @@ def resident_footprint(gguf: Path, budget: HardwareBudget, window: int, *, draft
return footprint_bytes(plan.profile, window, overhead_bytes=plan.overhead_bytes)
-def _launch_footprint(gguf: Path, budget: HardwareBudget) -> int | None:
+def _launch_footprint(gguf: Path, budget: HardwareBudget, engine_tag: str | None = None) -> int | None:
"""Estimated resident bytes for one staged model at the window this policy grants it, or None
when it cannot be priced: an unreadable header, or a model the physics check refuses outright
(it never loads, so it must not shrink the residency cap)."""
@@ -300,20 +300,28 @@ def _launch_footprint(gguf: Path, budget: HardwareBudget) -> int | None:
return None
entry = entry_for_model(model_id)
companions = _companions(entry)
+ fixed_overhead = RUNTIME_OVERHEAD_BYTES + companions.nbytes
+ requested = load_window_overrides().get(model_id)
plan = plan_launch(profile, budget, mtp_capable=_runs_mtp(entry, companions),
- fixed_overhead=RUNTIME_OVERHEAD_BYTES + companions.nbytes,
- requested_window=load_window_overrides().get(model_id))
+ fixed_overhead=fixed_overhead, requested_window=requested)
if isinstance(plan.decision, PhysicsRefusal):
return None
# Priced whole even when the plan spills: a spilled model still holds part of its weights on
- # the device, and over-counting errs toward the side that cannot thrash. A DFlash draft on
- # disk counts whether or not this machine runs it, for the same reason.
- draft = _dflash_extra_bytes(entry, profile) if _runs_dflash(entry) is not None else 0
- return footprint_bytes(plan.profile, plan.decision.window, overhead_bytes=plan.overhead_bytes) + draft
+ # the device, and over-counting errs toward the side that cannot thrash.
+ price = footprint_bytes(plan.profile, plan.decision.window, overhead_bytes=plan.overhead_bytes)
+ # Where the engine loads the draft, the preset runs it in place of MTP: the model holds the
+ # larger of the two launches, never both.
+ dflash = None if plan.decision.spilled else _dflash_plan(
+ entry, profile, budget, None, fixed_overhead=fixed_overhead, requested_window=requested,
+ engine_tag=engine_tag, window_without=plan.decision.window)
+ if dflash is not None:
+ price = max(price, footprint_bytes(dflash.profile, dflash.decision.window,
+ overhead_bytes=dflash.overhead_bytes))
+ return price
def admitted_residency_count(models_dir: Path, budget: HardwareBudget, configured: int,
- live: HardwareBudget | None = None) -> int:
+ live: HardwareBudget | None = None, engine_tag: str | None = None) -> int:
"""How many models the card may hold resident at once: priced against the budget, not a count.
Residency used to be bounded by a count alone, so a second model was admitted against an
@@ -335,7 +343,7 @@ def admitted_residency_count(models_dir: Path, budget: HardwareBudget, configure
return configured
largest = 0
for gguf in staged_in(models_dir):
- need = _launch_footprint(gguf, budget)
+ need = _launch_footprint(gguf, budget, engine_tag)
if need:
largest = max(largest, need)
if largest <= 0:
diff --git a/tests/hermes_cli/test_local_external_draft.py b/tests/hermes_cli/test_local_external_draft.py
index ff2510db7f5..c6a9263fc45 100644
--- a/tests/hermes_cli/test_local_external_draft.py
+++ b/tests/hermes_cli/test_local_external_draft.py
@@ -221,12 +221,31 @@ def test_a_model_without_mtp_drafts_without_backend_sampling(model):
assert "backend-sampling" not in preset.keys
-def test_the_residency_price_includes_the_draft_once_it_is_on_disk(model):
- gguf, draft, _ = model
- with_draft = presets._launch_footprint(gguf, ROOMY)
- draft.unlink()
+def test_the_residency_price_is_the_launch_the_model_runs(model):
+ """The preset runs the draft in place of MTP, so the model holds the larger of the two launches,
+ never both added together. An engine that can't load the draft prices MTP alone."""
+ gguf, _, use = model
+ larger = set()
+ for state in (150 * MIB, 600 * MIB): # MTP's own layers outweigh the draft, then the reverse
+ use(state=state)
+ mtp = presets.resident_footprint(gguf, ROOMY, NATIVE)
+ drafting = presets.resident_footprint(gguf, ROOMY, NATIVE, drafting=True)
+ larger.add("draft" if drafting > mtp else "mtp")
+
+ assert presets._launch_footprint(gguf, ROOMY, engine_tag="b11671") == max(mtp, drafting)
+ assert presets._launch_footprint(gguf, ROOMY, engine_tag="b11000") == mtp
+ assert presets._launch_footprint(gguf, ROOMY) == mtp
+ assert larger == {"draft", "mtp"}
+
+
+def test_two_models_stay_resident_where_two_drafted_launches_fit(model):
+ gguf, _, _ = model
+ gguf.touch()
+ held = max(presets.resident_footprint(gguf, ROOMY, NATIVE),
+ presets.resident_footprint(gguf, ROOMY, NATIVE, drafting=True))
+ card = replace(ROOMY, usable_vram_bytes=2 * held + GIB)
- assert with_draft >= presets._launch_footprint(gguf, ROOMY) + ENTRY.draft.size_bytes
+ assert presets.admitted_residency_count(gguf.parent, card, 4, engine_tag="b11671") == 2
def _staged(monkeypatch, *entries):
diff --git a/tests/hermes_cli/test_local_runtime.py b/tests/hermes_cli/test_local_runtime.py
index b9ef4cf13ca..4ccb8ad7be7 100644
--- a/tests/hermes_cli/test_local_runtime.py
+++ b/tests/hermes_cli/test_local_runtime.py
@@ -1010,7 +1010,9 @@ def test_boot_prices_the_engine_it_serves(tmp_path, monkeypatch):
return path
monkeypatch.setattr(bootstrap, "_generate_presets", _priced)
- monkeypatch.setattr(bootstrap, "_admitted_models_max", lambda mdir, configured: configured)
+ capped: list = []
+ monkeypatch.setattr(bootstrap, "_admitted_models_max",
+ lambda mdir, configured, engine_tag=None: capped.append(engine_tag) or configured)
class _NoStart:
def __init__(self, *a, **k):
@@ -1027,4 +1029,5 @@ def test_boot_prices_the_engine_it_serves(tmp_path, monkeypatch):
assert bootstrap.ensure_local_runtime({"local_runtime": {"enabled": True, "backend": "vulkan"}}) is None
# The release reaches the presets too: drafts gate on the engine that will load them.
assert priced == [(vulkan, "b1")]
+ assert capped == ["b1"] # the residency cap prices the draft only where this engine loads it
assert bootstrap.serving_engine() is None # a failed boot serves nothing
diff --git a/tests/hermes_cli/test_local_windows_commit_budget.py b/tests/hermes_cli/test_local_windows_commit_budget.py
index feab6721b95..d47b4fa5c1f 100644
--- a/tests/hermes_cli/test_local_windows_commit_budget.py
+++ b/tests/hermes_cli/test_local_windows_commit_budget.py
@@ -103,7 +103,7 @@ def test_router_holds_only_the_models_free_commit_fits(tmp_path, monkeypatch):
from hermes_cli.local_runtime import bootstrap
monkeypatch.setattr(bootstrap, "staged_in", lambda mdir: [tmp_path / "a.gguf", tmp_path / "b.gguf"])
- monkeypatch.setattr(presets, "_launch_footprint", lambda gguf, budget: 12 * G)
+ monkeypatch.setattr(presets, "_launch_footprint", lambda gguf, budget, engine_tag=None: 12 * G)
capacity = HardwareBudget(int(30 * G), int(50 * G), 0, uma=True)
live = HardwareBudget(int(16 * G), int(50 * G), 0, uma=True, commit_room_bytes=int(20 * G))
assert presets.admitted_residency_count(tmp_path, capacity, 4) == 2