Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Release GGUF context while tool approval is pending (#10673)

Lee Jacksonc2bfd2197033
diff --git a/studio/backend/core/inference/llama_admission.py b/studio/backend/core/inference/llama_admission.py
index 8653d3f3bbe..e600028add9 100644
--- a/studio/backend/core/inference/llama_admission.py
+++ b/studio/backend/core/inference/llama_admission.py
@@ -312,6 +312,8 @@ class LlamaAdmissionLease:
"_parked",
"_budgeted",
"_tokens",
+ "_parked_tokens",
+ "_park_id",
)
def __init__(
@@ -326,9 +328,9 @@ class LlamaAdmissionLease:
self._release_lock = threading.Lock()
self._parked = False
self._budgeted = False
- # Returned to the queue on release, not on park: a parked holder has stopped decoding but llama-server still
- # holds its KV until the task ends.
self._tokens = max(0, int(tokens or 0))
+ self._parked_tokens = 0
+ self._park_id = None
@property
def slot(self) -> Optional[int]:
@@ -338,6 +340,9 @@ class LlamaAdmissionLease:
def park(self) -> bool:
"""Hand the slot back while this holder waits on something off the GPU.
+ Tokens stay committed: llama-server holds this holder's KV until something erases it.
+ See ``release_parked_cache``.
+
A run stopped on a tool approval prompt is not decoding, so holding its slot would let
unanswered prompts fill the pool while llama-server idles. The lease itself stays valid:
releasing it after a park is still correct.
@@ -352,13 +357,40 @@ class LlamaAdmissionLease:
return False
# Under the lease lock so the decision and the handover cannot split. Nothing takes the queue lock then a
# lease lock, so this order is the only one in play.
- if not queue.try_park(self._slot):
+ park_id = queue.try_park(self._slot, tokens = self._tokens)
+ if park_id is None:
return False
+ self._park_id = park_id
self._parked = True
self._budgeted = True
self._slot = None
return True
+ @property
+ def parked_tokens(self) -> int:
+ with self._release_lock:
+ if self._released or not self._parked:
+ return 0
+ return self._tokens
+
+ def reclaim_would_admit(self) -> bool:
+ queue = self._queue
+ tokens = self.parked_tokens
+ if queue is None or tokens <= 0:
+ return False
+ return queue.reclaim_would_admit(tokens)
+
+ def release_parked_cache(self) -> int:
+ """Hand back a parked holder's commitment, once llama-server confirms the erasure."""
+ queue = self._queue
+ with self._release_lock:
+ if queue is None or self._released or not self._parked or self._tokens <= 0:
+ return 0
+ returned, self._tokens = self._tokens, 0
+ self._parked_tokens += returned
+ queue.reclaim_parked_tokens(returned, park_id = self._park_id)
+ return returned
+
def _drop_budget(self) -> None:
"""Give the executor budget back now the prompt wait is over.
@@ -384,22 +416,26 @@ class LlamaAdmissionLease:
if not self._parked:
return
self._parked = False
+ park_id = self._park_id
self._drop_budget()
if self._queue is not None:
- self._queue.unpark()
+ self._queue.unpark(park_id)
async def unpark_async(
self,
*,
cancel_event = None,
poll_s: float = 0.02,
+ timeout_s: Optional[float] = DEFAULT_RECOST_WAIT_TIMEOUT_S,
) -> None:
"""Take a slot back, waiting until the pool has room.
``park`` gave the slot to a waiter, so by the time the user answers the
prompt someone else may be decoding in it. Resuming regardless put two
- holders on a one-slot server. Gives up if the caller is cancelled, since
- the holder is then leaving anyway and must not be stuck here.
+ holders on a one-slot server. A holder whose cache was erased must win its
+ commitment back too, or it decodes against room the queue gave away. Gives
+ up if the caller is cancelled, since the holder is then leaving anyway and
+ must not be stuck here.
"""
queue = self._queue
if queue is None or not self._parked:
@@ -407,7 +443,15 @@ class LlamaAdmissionLease:
# Before the wait, not after: the prompt is answered, so this holder is already off the executor and must not
# keep anyone else off it.
self._drop_budget()
- slot = await queue.acquire_parked_slot(cancel_event = cancel_event, poll_s = poll_s)
+ tokens = self._parked_tokens
+ park_id = self._park_id
+ slot = await queue.acquire_parked_slot(
+ cancel_event = cancel_event,
+ poll_s = poll_s,
+ tokens = tokens,
+ timeout_s = timeout_s,
+ abandoned = lambda: self._released,
+ )
stranded = None
with self._release_lock:
# release() may have run during the wait; it clears the flag and does the unpark itself, so only the caller
@@ -419,10 +463,13 @@ class LlamaAdmissionLease:
stranded = slot
else:
self._slot = slot
+ if slot is not None:
+ self._tokens += tokens
+ self._parked_tokens = 0
if parked:
- queue.unpark()
+ queue.unpark(park_id)
if stranded is not None:
- queue.release(stranded)
+ queue.release(stranded, tokens)
def recost(self, tokens: int) -> bool:
"""Re-state what this lease actually occupies as its conversation grows.
@@ -543,10 +590,11 @@ class LlamaAdmissionLease:
self._released = True
queue = self._queue
parked, self._parked = self._parked, False
+ park_id = self._park_id
self._drop_budget()
if queue is not None:
if parked:
- queue.unpark()
+ queue.unpark(park_id)
queue.release(self._slot, self._tokens)
async def __aenter__(self) -> "LlamaAdmissionLease":
@@ -655,7 +703,11 @@ class LlamaAdmissionQueue:
"_waiters",
"_parked",
"_unpark_tickets",
+ "_unpark_tokens",
+ "_unpark_lapsed",
"_unpark_seq",
+ "_parked_reclaimable",
+ "_park_seq",
"_committed",
"_budget",
"_reparking",
@@ -677,7 +729,11 @@ class LlamaAdmissionQueue:
# FIFO tickets for holders resuming from a park (see acquire_parked_slot). A bare count deadlocked: every
# approved holder blocked every other one.
self._unpark_tickets: Deque[int] = deque()
+ self._unpark_tokens: dict[int, int] = {}
+ self._unpark_lapsed: set = set()
self._unpark_seq = 0
+ self._parked_reclaimable: dict[int, int] = {}
+ self._park_seq = 0
# KV tokens held by live leases, against the cache size the caller reports. 0 budget disables the check, which
# is what every pre-existing caller gets.
self._committed = 0
@@ -693,8 +749,8 @@ class LlamaAdmissionQueue:
self._capacity = capacity
self._free = [slot for slot in range(capacity) if not self._in_use >> slot & 1]
- def _fits_budget_locked(self, tokens: int) -> bool:
- """Whether ``tokens`` more KV may be committed.
+ def _fits_against_locked(self, committed: int, tokens: int) -> bool:
+ """Whether ``tokens`` fit a cache already holding ``committed``.
A caller is always admitted when nothing else holds KV, however large it is. Its request may
still be refused by llama-server, but that refusal names both token counts and the setting
@@ -709,14 +765,30 @@ class LlamaAdmissionQueue:
"""
if self._budget <= 0 or tokens <= 0:
return True
- if self._committed == 0:
+ if committed <= 0:
return True
- return self._committed + tokens <= self._budget
+ return committed + tokens <= self._budget
+
+ def _fits_budget_locked(
+ self,
+ tokens: int,
+ reserved_tokens: int = 0,
+ ) -> bool:
+ """Whether ``tokens`` more KV fit over the ``reserved_tokens`` owed to resumes."""
+ return self._fits_against_locked(self._committed + max(0, reserved_tokens), tokens)
+
+ def _reserved_tokens_locked(self) -> int:
+ return sum(
+ tokens
+ for ticket, tokens in self._unpark_tokens.items()
+ if ticket not in self._unpark_lapsed
+ )
def _can_admit_locked(
self,
reserved: int,
tokens: int = 0,
+ reserved_tokens: int = 0,
) -> bool:
# Slots still held above a shrunk capacity keep occupying the backend, so count every held slot against the
# ceiling, not just the ids below it. ``reserved`` holds slots back for approved holders waiting to resume;
@@ -725,14 +797,15 @@ class LlamaAdmissionQueue:
return False
# A free slot is not enough: with --kv-unified every slot reports the full n_ctx, so the pool can hand out more
# slots than the one cache can serve.
- return self._fits_budget_locked(tokens)
+ return self._fits_budget_locked(tokens, reserved_tokens)
def _take_slot_locked(
self,
reserved: int,
tokens: int = 0,
+ reserved_tokens: int = 0,
) -> Optional[int]:
- if not self._can_admit_locked(reserved, tokens):
+ if not self._can_admit_locked(reserved, tokens, reserved_tokens):
return None
slot = self._free.pop()
self._in_use |= 1 << slot
@@ -777,7 +850,9 @@ class LlamaAdmissionQueue:
# for more, so an arrival admitted here would take exactly that, pinning a growing conversation at its
# opening size for as long as traffic lasts.
if not self._waiters and self._reparking == 0:
- slot = self._take_slot_locked(len(self._unpark_tickets), cost)
+ slot = self._take_slot_locked(
+ len(self._unpark_tickets), cost, self._reserved_tokens_locked()
+ )
if slot is not None:
# No snapshot here: callers read it through snapshot_now(), which re-reads the queue, so building
# one per admitted request would be pure allocation on the hot path.
@@ -904,56 +979,144 @@ class LlamaAdmissionQueue:
self._committed += max(0, int(restore or 0))
self._grant_waiters_locked()
- def try_park(self, slot: Optional[int]) -> bool:
+ def try_park(
+ self,
+ slot: Optional[int],
+ *,
+ tokens: int = 0,
+ ) -> Optional[int]:
"""Return a parked holder's slot to the pool. See ``LlamaAdmissionLease.park``.
- False leaves the slot with its holder, so a refused park costs nothing to
- undo. The per-queue count is only what ``is_idle`` reads; the budget and
- the capacity it is sized from are both process-wide.
+ None leaves the slot with its holder, so a refused park costs nothing to
+ undo. Otherwise an id identifying this park until the holder leaves it. The
+ per-queue count is only what ``is_idle`` reads; the budget and the capacity it
+ is sized from are both process-wide.
+
+ ``tokens`` stays committed, and is recorded only so ``reclaim_would_admit``
+ knows how much erasing could free.
"""
if not _claim_park(_max_parked(_live_capacity(self))):
- return False
+ return None
with self._lock:
self._parked += 1
+ self._park_seq += 1
+ park_id = self._park_seq
+ self._parked_reclaimable[park_id] = max(0, int(tokens or 0))
self._release_slot_locked(slot)
self._grant_waiters_locked()
- return True
+ return park_id
- def unpark(self) -> None:
+ def reclaim_parked_tokens(
+ self,
+ tokens: int,
+ *,
+ park_id: Optional[int] = None,
+ ) -> None:
+ with self._lock:
+ self._parked_reclaimable.pop(park_id, None)
+ self._committed = max(0, self._committed - max(0, int(tokens or 0)))
+ self._grant_waiters_locked()
+
+ def _ticket_position_locked(self, ticket: int) -> tuple:
+ """(slots, room) that resuming holders ahead of ``ticket`` hold back from it.
+
+ One ahead reserves room only until its deadline lapses, but holds a slot back only
+ while it can afford that room: pinning one it cannot pay for deadlocks the pool.
+ """
+ ahead = 0
+ ahead_tokens = 0
+ for queued in self._unpark_tickets:
+ if queued == ticket:
+ break
+ needs = self._unpark_tokens.get(queued, 0)
+ if self._fits_budget_locked(needs, ahead_tokens):
+ ahead += 1
+ if queued not in self._unpark_lapsed:
+ ahead_tokens += needs
+ return (ahead, ahead_tokens)
+
+ def _claimants_locked(self):
+ for ticket in self._unpark_tickets:
+ yield (self._unpark_tokens.get(ticket, 0), *self._ticket_position_locked(ticket))
+ self._prune_waiters_locked()
+ if self._waiters:
+ yield (
+ self._waiters[0].tokens,
+ len(self._unpark_tickets),
+ self._reserved_tokens_locked(),
+ )
+
+ def reclaim_would_admit(self, tokens: int) -> bool:
+ """Whether erasing parked caches would let anyone admission owes into the cache.
+
+ Weighed in aggregate: one claimant erasing cannot fit must not hide a smaller one,
+ nor two too-small holders both decline and leave the queue stalled.
+ """
+ if max(0, int(tokens or 0)) <= 0:
+ return False
+ with self._lock:
+ if self._budget <= 0 or self._reparking > 0:
+ return False
+ reclaimable = sum(self._parked_reclaimable.values())
+ for wanted, held_back, reserved in self._claimants_locked():
+ if self._can_admit_locked(held_back, wanted, reserved):
+ continue
+ if not (bool(self._free) and (self._held + held_back) < self._capacity):
+ continue
+ if self._fits_against_locked(
+ max(0, self._committed - reclaimable) + reserved, wanted
+ ):
+ return True
+ return False
+
+ def unpark(self, park_id: Optional[int] = None) -> None:
with self._lock:
if self._parked > 0:
self._parked -= 1
+ self._parked_reclaimable.pop(park_id, None)
async def acquire_parked_slot(
self,
*,
cancel_event = None,
poll_s: float = 0.02,
+ tokens: int = 0,
+ timeout_s: Optional[float] = DEFAULT_RECOST_WAIT_TIMEOUT_S,
+ abandoned = None,
) -> Optional[int]:
- """Wait for a slot for a holder resuming from a park, None if cancelled.
+ """Wait for a slot, and for ``tokens`` of cache room, None if cancelled.
Ordered by ticket rather than counted, so approvals resume in the order
they came back: counting them made every approved holder block every
other one, and with nothing decoding that never resolved.
+
+ ``tokens`` is what a holder whose cache was erased must win back, held from arrivals
+ until ``timeout_s`` lapses the claim. Lapsing keeps the ticket's place in the slot
+ order and still never admits it over budget: llama.cpp answers an exhausted cache by
+ clearing every slot it is decoding.
"""
+ tokens = max(0, int(tokens or 0))
with self._lock:
self._unpark_seq += 1
ticket = self._unpark_seq
self._unpark_tickets.append(ticket)
+ self._unpark_tokens[ticket] = tokens
+ deadline = None if not timeout_s or timeout_s <= 0 else time.monotonic() + timeout_s
try:
while True:
with self._lock:
- ahead = 0
- for queued in self._unpark_tickets:
- if queued == ticket:
- break
- ahead += 1
- # Only the approvals ahead of this one hold slots back from it.
- slot = self._take_slot_locked(ahead)
+ if deadline is not None and time.monotonic() >= deadline:
+ self._unpark_lapsed.add(ticket)
+ deadline = None
+ self._grant_waiters_locked()
+ ahead, ahead_tokens = self._ticket_position_locked(ticket)
+ slot = self._take_slot_locked(ahead, tokens, ahead_tokens)
if slot is not None:
return slot
if cancel_event is not None and cancel_event.is_set():
return None
+ if abandoned is not None and abandoned():
+ return None
await asyncio.sleep(poll_s)
finally:
with self._lock:
@@ -961,6 +1124,8 @@ class LlamaAdmissionQueue:
self._unpark_tickets.remove(ticket)
except ValueError:
pass
+ self._unpark_tokens.pop(ticket, None)
+ self._unpark_lapsed.discard(ticket)
self._grant_waiters_locked()
def cancel(self, waiter: _Waiter) -> None:
@@ -1007,12 +1172,14 @@ class LlamaAdmissionQueue:
# traffic keeps arriving. Head-of-line blocking is the fair trade here, and it matches the FIFO the rest of this
# queue already promises.
while self._waiters and self._can_admit_locked(
- len(self._unpark_tickets), self._waiters[0].tokens
+ len(self._unpark_tickets), self._waiters[0].tokens, self._reserved_tokens_locked()
):
waiter = self._waiters.popleft()
if waiter.cancelled or waiter.future.done():
continue
- slot = self._take_slot_locked(len(self._unpark_tickets), waiter.tokens)
+ slot = self._take_slot_locked(
+ len(self._unpark_tickets), waiter.tokens, self._reserved_tokens_locked()
+ )
lease = LlamaAdmissionLease(self, slot, waiter.tokens)
waiter.granted_lease = lease
try:
diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.py
index ffc3cfec83d..1f222636590 100644
--- a/studio/backend/core/inference/llama_cpp.py
+++ b/studio/backend/core/inference/llama_cpp.py
@@ -31608,6 +31608,56 @@ class LlamaCppBackend:
continue
raise
+ @staticmethod
+ def decode_slot_from_chunk(chunk) -> Optional[int]:
+ """The llama-server slot a stream chunk was decoded on, None if it does not say.
+
+ Only the final result carries ``__verbose``, and only when asked for.
+ """
+ if not isinstance(chunk, dict):
+ return None
+ verbose = chunk.get("__verbose")
+ if not isinstance(verbose, dict):
+ return None
+ slot = verbose.get("id_slot")
+ if type(slot) is not int or slot < 0:
+ return None
+ return slot
+
+ def release_idle_chat_slot(self, base_url: str, slot: int) -> bool:
+ """Erase one completed round's cached context, True once the engine says so.
+
+ Declines unless the round decoded on this server, which a reload renumbers, and
+ unless --slot-save-path is in play, which the endpoint requires. An erase aimed at
+ a slot since handed to another request is deferred rather than dropped, so even a
+ timeout can cost that chat its prefix.
+ """
+ if (
+ not self._slot_save_dir
+ or base_url != self.base_url
+ or type(slot) is not int
+ or slot < 0
+ ):
+ return False
+ try:
+ response = httpx.post(
+ f"{base_url}/slots/{slot}",
+ params = {"action": "erase"},
+ headers = self._auth_headers,
+ timeout = 2.0,
+ trust_env = False,
+ )
+ response.raise_for_status()
+ result = response.json()
+ return (
+ result.get("id_slot") == slot
+ and type(result.get("n_erased")) is int
+ and result["n_erased"] >= 0
+ )
+ except Exception:
+ logger.debug("Approval cache reclamation failed; retaining reservation", exc_info = True)
+ return False
+
def generate_chat_completion(
self,
messages: list[dict],
@@ -32056,6 +32106,7 @@ class LlamaCppBackend:
# MAY BLOCK: recost_waiting waits for cache room. Safe at the top of a round,
# where the previous round's request has completed.
on_conversation_grew: Optional[Callable[[list], None]] = None,
+ on_decode_slot: Optional[Callable[[str, int], None]] = None,
) -> Generator[dict, None, None]:
"""
Agentic loop: let the model call tools, execute them, and continue.
@@ -32774,6 +32825,9 @@ class LlamaCppBackend:
# Progress events feed the first-token deadline; timings stay opt-in.
payload["return_progress"] = True
+ if on_decode_slot is not None:
+ payload["verbose"] = True
+ payload["response_fields"] = ["id_slot"]
if perf_callback is not None:
payload["timings_per_token"] = True
if logit_bias:
@@ -33057,6 +33111,10 @@ class LlamaCppBackend:
try:
chunk_data = json.loads(line[6:])
+ if on_decode_slot is not None:
+ slot = self.decode_slot_from_chunk(chunk_data)
+ if slot is not None:
+ on_decode_slot(self.base_url, slot)
_report_live_llama_timings(perf_callback, chunk_data)
_ct = chunk_data.get("timings")
diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py
index 03dea9f0f3c..c8ad552757f 100644
--- a/studio/backend/routes/inference.py
+++ b/studio/backend/routes/inference.py
@@ -1748,6 +1748,7 @@ _TEARDOWN_TASK_STOP_TIMEOUT_S = 5.0
# (e.g. prompt prefill between tool iterations). A second layer atop the
# tool_stream_exec heartbeats, keeping proxies (Cloudflare drops idle at ~100s).
_LOCAL_TOOL_STREAM_STALL_KEEPALIVE_S = 15.0
+_APPROVAL_CACHE_RECLAIM_POLL_S = 0.25
def _openai_llama_admission_capacity(request: Optional[Request], llama_backend = None) -> int:
@@ -2225,6 +2226,7 @@ def _openai_llama_admission_recost(
output_tokens: Optional[int] = None,
cancel_event = None,
injected_tools = None,
+ cache_is_empty: bool = False,
) -> None:
"""Charge a tool loop for what its conversation now is, not what it opened as.
@@ -2241,6 +2243,10 @@ def _openai_llama_admission_recost(
Safe to wait here because it is between rounds and the slot is idle at llama-server.
Idle is not reclaimed, though, so the yield is gated on
``_openai_llama_admission_can_yield``; where it is False this declines instead.
+
+ ``cache_is_empty`` overrides that gate for a round whose cells were explicitly erased.
+ The gate exists because an idle slot's KV stays resident, which an erased slot's does
+ not, so yielding there hands back room that really is free.
"""
if reservation is None:
return
@@ -2287,7 +2293,7 @@ def _openai_llama_admission_recost(
lease.recost_waiting(
want,
cancel_event = cancel_event,
- allow_yield = _openai_llama_admission_can_yield(llama_backend),
+ allow_yield = cache_is_empty or _openai_llama_admission_can_yield(llama_backend),
)
except Exception: # pragma: no cover - accounting must not break a live run
logger.debug("llama admission recost failed", exc_info = True)
@@ -23459,8 +23465,19 @@ async def produce_openai_chat_completions(
# reservation exists but not ITERATED until after, so the callback always sees
# a reservation by the time a round can call it.
_gguf_admission_hold: dict = {"reservation": None}
+ _gguf_decode_lock = threading.Lock()
+ _gguf_decode: dict = {"slot": None, "erased": False}
+
+ def _gguf_record_decode_slot(base_url: str, slot: int) -> None:
+ with _gguf_decode_lock:
+ _gguf_decode["slot"] = (base_url, slot)
+ _gguf_decode["erased"] = False
def _gguf_recost(conversation) -> None:
+ with _gguf_decode_lock:
+ erased = _gguf_decode["erased"]
+ _gguf_decode["slot"] = None
+ _gguf_decode["erased"] = False
_openai_llama_admission_recost(
_gguf_admission_hold["reservation"],
conversation,
@@ -23477,6 +23494,8 @@ async def produce_openai_chat_completions(
# A round waiting for cache room must still answer Stop. Same event
# the loop polls each iteration, so a wait ends where a cancel would.
cancel_event = cancel_event,
+ # Erased cells make yielding honest, so a grown round may wait.
+ cache_is_empty = erased,
)
# Active tool names gating the bare-rehearsal strip, matching the loop gate.
@@ -23542,6 +23561,10 @@ async def produce_openai_chat_completions(
permission_mode = payload.permission_mode,
perf_callback = _gguf_perf_callback,
on_conversation_grew = _gguf_recost,
+ # Only the streaming path parks and reclaims, so only it can use a slot.
+ on_decode_slot = _gguf_record_decode_slot
+ if payload.stream and _effective_confirm and not payload.bypass_permissions
+ else None,
context_overflow = _rolling_context_policy(payload),
context_policy = _request_context_policy(payload),
compaction_headroom_ratio = _request_compaction_headroom_ratio(payload),
@@ -23594,6 +23617,77 @@ async def produce_openai_chat_completions(
# otherwise unanswered prompts hold every slot.
_parked = False
+ _reclaim_task = None
+ _reclaim_stop = asyncio.Event()
+
+ def _erase_and_record(target) -> bool:
+ if not llama_backend.release_idle_chat_slot(*target):
+ return False
+ with _gguf_decode_lock:
+ _gguf_decode["erased"] = True
+ return True
+
+ async def _reclaim_approval_cache(lease):
+ """Erase this round's cached context once a queued chat needs its room.
+
+ On demand only: the engine drops the cells rather than spilling them,
+ so this costs the approved chat a full reprocess. Stopping is honoured
+ only before the erase is sent; past that the cells are going regardless.
+ """
+ try:
+ while not lease.reclaim_would_admit():
+ try:
+ await asyncio.wait_for(
+ _reclaim_stop.wait(), _APPROVAL_CACHE_RECLAIM_POLL_S
+ )
+ except asyncio.TimeoutError:
+ continue
+ return
+ with _gguf_decode_lock:
+ target = _gguf_decode["slot"]
+ erased = _gguf_decode["erased"]
+ if target is None or _reclaim_stop.is_set():
+ return
+ if not erased and not await asyncio.to_thread(_erase_and_record, target):
+ return
+ lease.release_parked_cache()
+ except asyncio.CancelledError:
+ raise
+ except Exception:
+ logger.debug("Approval cache reclamation failed", exc_info = True)
+
+ def _watch_for_reclaim(lease) -> None:
+ nonlocal _reclaim_task
+ with _gguf_decode_lock:
+ known = _gguf_decode["slot"] is not None
+ if not known or _reclaim_task is not None:
+ return
+ _reclaim_stop.clear()
+ _reclaim_task = asyncio.create_task(_reclaim_approval_cache(lease))
+
+ async def _settle_reclaim_watch() -> None:
+ """Wait out a reclamation that is already sending, before resuming.
+
+ Cancelling would not stop it: the worker runs on and the erase lands
+ anyway, leaving the resumed round priced against cells mid-drop.
+ """
+ nonlocal _reclaim_task
+ task, _reclaim_task = _reclaim_task, None
+ if task is None:
+ return
+ _reclaim_stop.set()
+ try:
+ await task
+ except Exception:
+ logger.debug("Approval cache reclamation failed", exc_info = True)
+
+ def _stop_reclaim_watch() -> None:
+ nonlocal _reclaim_task
+ _reclaim_stop.set()
+ if _reclaim_task is not None:
+ _reclaim_task.cancel()
+ _reclaim_task = None
+
async def _park_admission(on: bool, *, wait: bool = True):
nonlocal _parked
if on == _parked:
@@ -23604,16 +23698,15 @@ async def produce_openai_chat_completions(
if lease is None:
return
if on:
- # Refused when the budget is spent: the slot stays here,
- # so there is nothing to take back afterwards.
if not lease.park():
return
+ _watch_for_reclaim(lease)
elif wait:
- # Resuming: park() may have handed our slot to a waiter, so wait for room instead
- # of putting two holders on one slot.
+ await _settle_reclaim_watch()
await lease.unpark_async(cancel_event = cancel_event)
else:
# Tearing down; the lease is released separately.
+ _stop_reclaim_watch()
lease.unpark()
_parked = on
diff --git a/studio/backend/tests/test_llama_admission_compat_matrix.py b/studio/backend/tests/test_llama_admission_compat_matrix.py
index 77155641382..b4b4688523d 100644
--- a/studio/backend/tests/test_llama_admission_compat_matrix.py
+++ b/studio/backend/tests/test_llama_admission_compat_matrix.py
@@ -184,6 +184,10 @@ class TestNothingChangesWhenTheBudgetIsUnknown:
assert queue._reparking == 0
+# In the order they were added, so older positional callers keep their meaning.
+_TOOL_LOOP_HOOKS = ("on_conversation_grew", "on_decode_slot")
+
+
class TestOldCallers:
"""Everything added is keyword-with-default, so code written before this still runs."""
@@ -220,8 +224,8 @@ class TestOldCallers:
from core.inference.llama_cpp import LlamaCppBackend
signature = inspect.signature(LlamaCppBackend.generate_chat_completion_with_tools)
- parameter = signature.parameters["on_conversation_grew"]
- assert parameter.default is None, "the hook must be optional for existing callers"
+ for name in _TOOL_LOOP_HOOKS:
+ assert signature.parameters[name].default is None, f"{name} must be optional"
def test_the_hook_was_appended_rather_than_inserted(self):
"""No bare ``*`` in this signature, so every parameter is positional-or-keyword and
@@ -234,9 +238,8 @@ class TestOldCallers:
names = list(
inspect.signature(LlamaCppBackend.generate_chat_completion_with_tools).parameters
)
- assert (
- names[-1] == "on_conversation_grew"
- ), f"the hook must be last; signature ends {names[-3:]}"
+ tail = names[-len(_TOOL_LOOP_HOOKS) :]
+ assert tail == list(_TOOL_LOOP_HOOKS), f"the hooks must stay at the tail, got {tail}"
def test_the_wait_timeout_has_a_sane_default(self):
import inspect
@@ -245,8 +248,9 @@ class TestOldCallers:
from core.inference.llama_admission import LlamaAdmissionLease
- signature = inspect.signature(LlamaAdmissionLease.recost_waiting)
- assert signature.parameters["timeout_s"].default == DEFAULT_RECOST_WAIT_TIMEOUT_S
+ for method in (LlamaAdmissionLease.recost_waiting, LlamaAdmissionLease.unpark_async):
+ signature = inspect.signature(method)
+ assert signature.parameters["timeout_s"].default == DEFAULT_RECOST_WAIT_TIMEOUT_S
class TestNoPersistentStateChanged:
diff --git a/studio/backend/tests/test_llama_cpp_tool_loop.py b/studio/backend/tests/test_llama_cpp_tool_loop.py
index 5ce59f06af4..43884f5f758 100644
--- a/studio/backend/tests/test_llama_cpp_tool_loop.py
+++ b/studio/backend/tests/test_llama_cpp_tool_loop.py
@@ -6304,6 +6304,27 @@ def test_the_synthesized_final_pass_is_recosted_before_it_is_sent(monkeypatch):
)
[email protected]("wanted", [True, False])
+def test_the_decode_slot_is_asked_for_and_read_only_when_a_caller_wants_it(monkeypatch, wanted):
+ """Verbose alone would attach the whole prompt, so it travels with the narrowing."""
+ final = {"choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}]}
+ final["__verbose"] = {"id_slot": 3}
+ stream = [_sse({"content": "done"}), "data: " + json.dumps(final) + "\n", _done()]
+ backend, payloads = _backend_and_payloads(monkeypatch, [stream])
+ seen = []
+
+ _run_tool_loop(
+ backend,
+ [{"role": "user", "content": "hi"}],
+ _render_html_tools(),
+ on_decode_slot = (lambda url, slot: seen.append((url, slot))) if wanted else None,
+ )
+
+ assert seen == ([(backend.base_url, 3)] if wanted else [])
+ assert payloads[0].get("verbose") is (True if wanted else None)
+ assert payloads[0].get("response_fields") == (["id_slot"] if wanted else None)
+
+
@pytest.mark.parametrize(
"held",
[
diff --git a/studio/backend/tests/test_tool_approval_context_release.py b/studio/backend/tests/test_tool_approval_context_release.py
new file mode 100644
index 00000000000..6eeedab6fe0
--- /dev/null
+++ b/studio/backend/tests/test_tool_approval_context_release.py
@@ -0,0 +1,115 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""A chat parked on a tool approval gives its cached context back on demand."""
+
+import asyncio
+from types import SimpleNamespace
+
+import pytest
+import core.inference.llama_cpp as llama_cpp
+from core.inference.llama_admission import (
+ LlamaAdmissionConfig,
+ LlamaAdmissionQueue,
+ reset_llama_admission_queues,
+)
+
+
[email protected](autouse = True)
+def reset():
+ reset_llama_admission_queues()
+ yield
+ reset_llama_admission_queues()
+
+
+def reserve(queue, tokens, capacity):
+ config = LlamaAdmissionConfig()
+ return queue.reserve(capacity = capacity, config = config, tokens = tokens, budget = 35000)
+
+
+def test_a_parked_holder_keeps_its_context_until_someone_is_blocked_on_it():
+ async def scenario():
+ queue = LlamaAdmissionQueue("demand")
+ parked = reserve(queue, 29000, 4).lease_nowait()
+ assert parked.park()
+ assert queue.snapshot().committed == 29000, "parking returns the slot, not the KV"
+ assert not parked.reclaim_would_admit(), "erasing costs a reprocess, admits nobody"
+ waiter = reserve(queue, 8750, 4)
+ assert waiter.lease_nowait() is None
+ assert parked.reclaim_would_admit()
+ waiter.cancel()
+ assert not parked.reclaim_would_admit()
+
+ asyncio.run(scenario())
+
+
+def test_a_resume_wins_its_room_back_and_is_not_overtaken():
+ async def scenario():
+ queue = LlamaAdmissionQueue("resume")
+ first = reserve(queue, 29000, 4).lease_nowait()
+ assert first.park()
+ assert first.release_parked_cache() == 29000
+ busy = reserve(queue, 20000, 4).lease_nowait() # blocks the resume, not a small arrival
+ resumed = asyncio.create_task(first.unpark_async(poll_s = 0.001))
+ await asyncio.sleep(0.01)
+ assert not resumed.done(), "a spare slot does not justify exceeding the KV budget"
+ arrivals = [reserve(queue, 8000, 4) for _ in range(4)]
+ assert not any(a.lease_nowait() for a in arrivals), "the approved chat is owed this"
+ busy.release()
+ await asyncio.wait_for(resumed, 1)
+ assert queue.snapshot().committed == 29000, "the queue drains to the resumer first"
+ for arrival in arrivals:
+ arrival.cancel()
+ first.release()
+ assert queue.snapshot().committed == 0, "what the resume took back is given back"
+
+ asyncio.run(scenario())
+
+
[email protected]("timeout_s", [None, 0.01])
+def test_a_claim_it_cannot_pay_for_does_not_pin_the_slot_a_later_one_needs(timeout_s):
+ """The first needs KV the second holds; the second needs the slot the first reserves."""
+
+ async def scenario():
+ queue = LlamaAdmissionQueue(f"deadlock-{timeout_s}")
+ first = reserve(queue, 10000, 1).lease_nowait()
+ assert first.park()
+ assert first.release_parked_cache() == 10000
+ second = reserve(queue, 35000, 1).lease_nowait()
+ assert second is not None, "the whole cache, freed by the first chat's erasure"
+ assert second.park()
+ ahead = asyncio.create_task(first.unpark_async(poll_s = 0.001, timeout_s = timeout_s))
+ await asyncio.sleep(0.05)
+ assert not ahead.done(), "the cache is committed; 10000 more does not fit"
+ await asyncio.wait_for(second.unpark_async(poll_s = 0.001), 1)
+ second.release()
+ await asyncio.wait_for(ahead, 1)
+ assert queue.snapshot().committed == 10000
+
+ asyncio.run(scenario())
+
+
+ "save_dir,other,body,ok",
+ [
+ ("/tmp/slots", False, {"id_slot": 2, "n_erased": 12}, True),
+ ("/tmp/slots", False, {"id_slot": 1, "n_erased": 12}, False),
+ ("/tmp/slots", False, {"id_slot": 2}, False),
+ ("/tmp/slots", True, None, False),
+ (None, False, None, False),
+ ],
+)
+def test_erasure_unconfirmed_keeps_the_context(monkeypatch, save_dir, other, body, ok):
+ server = SimpleNamespace(base_url = "http://s", _auth_headers = {}, _slot_save_dir = save_dir)
+
+ sent = []
+
+ def post(url, **kwargs):
+ sent.append((url, kwargs["params"]))
+ return SimpleNamespace(raise_for_status = lambda: None, json = lambda: body)
+
+ monkeypatch.setattr(llama_cpp.httpx, "post", post)
+ target = "http://other" if other else server.base_url
+ assert llama_cpp.LlamaCppBackend.release_idle_chat_slot(server, target, 2) is ok
+ # A swallowed request reads as a refusal, so the call itself is the assertion.
+ assert sent == ([] if body is None else [("http://s/slots/2", {"action": "erase"})])