Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Release GGUF context while tool approval is pending (#10673)
Lee Jacksonc2bfd2197033
diff --git a/studio/backend/core/inference/llama_admission.py b/studio/backend/core/inference/llama_admission.pyindex 8653d3f3bbe..e600028add9 100644--- a/studio/backend/core/inference/llama_admission.py+++ b/studio/backend/core/inference/llama_admission.py@@ -312,6 +312,8 @@ class LlamaAdmissionLease:"_parked","_budgeted","_tokens",+ "_parked_tokens",+ "_park_id",)def __init__(@@ -326,9 +328,9 @@ class LlamaAdmissionLease:self._release_lock = threading.Lock()self._parked = Falseself._budgeted = False- # Returned to the queue on release, not on park: a parked holder has stopped decoding but llama-server still- # holds its KV until the task ends.self._tokens = max(0, int(tokens or 0))+ self._parked_tokens = 0+ self._park_id = None@propertydef slot(self) -> Optional[int]:@@ -338,6 +340,9 @@ class LlamaAdmissionLease:def park(self) -> bool:"""Hand the slot back while this holder waits on something off the GPU.+ Tokens stay committed: llama-server holds this holder's KV until something erases it.+ See ``release_parked_cache``.+A run stopped on a tool approval prompt is not decoding, so holding its slot would letunanswered prompts fill the pool while llama-server idles. The lease itself stays valid:releasing it after a park is still correct.@@ -352,13 +357,40 @@ class LlamaAdmissionLease:return False# Under the lease lock so the decision and the handover cannot split. Nothing takes the queue lock then a# lease lock, so this order is the only one in play.- if not queue.try_park(self._slot):+ park_id = queue.try_park(self._slot, tokens = self._tokens)+ if park_id is None:return False+ self._park_id = park_idself._parked = Trueself._budgeted = Trueself._slot = Nonereturn True+ @property+ def parked_tokens(self) -> int:+ with self._release_lock:+ if self._released or not self._parked:+ return 0+ return self._tokens++ def reclaim_would_admit(self) -> bool:+ queue = self._queue+ tokens = self.parked_tokens+ if queue is None or tokens <= 0:+ return False+ return queue.reclaim_would_admit(tokens)++ def release_parked_cache(self) -> int:+ """Hand back a parked holder's commitment, once llama-server confirms the erasure."""+ queue = self._queue+ with self._release_lock:+ if queue is None or self._released or not self._parked or self._tokens <= 0:+ return 0+ returned, self._tokens = self._tokens, 0+ self._parked_tokens += returned+ queue.reclaim_parked_tokens(returned, park_id = self._park_id)+ return returned+def _drop_budget(self) -> None:"""Give the executor budget back now the prompt wait is over.@@ -384,22 +416,26 @@ class LlamaAdmissionLease:if not self._parked:returnself._parked = False+ park_id = self._park_idself._drop_budget()if self._queue is not None:- self._queue.unpark()+ self._queue.unpark(park_id)async def unpark_async(self,*,cancel_event = None,poll_s: float = 0.02,+ timeout_s: Optional[float] = DEFAULT_RECOST_WAIT_TIMEOUT_S,) -> None:"""Take a slot back, waiting until the pool has room.``park`` gave the slot to a waiter, so by the time the user answers theprompt someone else may be decoding in it. Resuming regardless put two- holders on a one-slot server. Gives up if the caller is cancelled, since- the holder is then leaving anyway and must not be stuck here.+ holders on a one-slot server. A holder whose cache was erased must win its+ commitment back too, or it decodes against room the queue gave away. Gives+ up if the caller is cancelled, since the holder is then leaving anyway and+ must not be stuck here."""queue = self._queueif queue is None or not self._parked:@@ -407,7 +443,15 @@ class LlamaAdmissionLease:# Before the wait, not after: the prompt is answered, so this holder is already off the executor and must not# keep anyone else off it.self._drop_budget()- slot = await queue.acquire_parked_slot(cancel_event = cancel_event, poll_s = poll_s)+ tokens = self._parked_tokens+ park_id = self._park_id+ slot = await queue.acquire_parked_slot(+ cancel_event = cancel_event,+ poll_s = poll_s,+ tokens = tokens,+ timeout_s = timeout_s,+ abandoned = lambda: self._released,+ )stranded = Nonewith self._release_lock:# release() may have run during the wait; it clears the flag and does the unpark itself, so only the caller@@ -419,10 +463,13 @@ class LlamaAdmissionLease:stranded = slotelse:self._slot = slot+ if slot is not None:+ self._tokens += tokens+ self._parked_tokens = 0if parked:- queue.unpark()+ queue.unpark(park_id)if stranded is not None:- queue.release(stranded)+ queue.release(stranded, tokens)def recost(self, tokens: int) -> bool:"""Re-state what this lease actually occupies as its conversation grows.@@ -543,10 +590,11 @@ class LlamaAdmissionLease:self._released = Truequeue = self._queueparked, self._parked = self._parked, False+ park_id = self._park_idself._drop_budget()if queue is not None:if parked:- queue.unpark()+ queue.unpark(park_id)queue.release(self._slot, self._tokens)async def __aenter__(self) -> "LlamaAdmissionLease":@@ -655,7 +703,11 @@ class LlamaAdmissionQueue:"_waiters","_parked","_unpark_tickets",+ "_unpark_tokens",+ "_unpark_lapsed","_unpark_seq",+ "_parked_reclaimable",+ "_park_seq","_committed","_budget","_reparking",@@ -677,7 +729,11 @@ class LlamaAdmissionQueue:# FIFO tickets for holders resuming from a park (see acquire_parked_slot). A bare count deadlocked: every# approved holder blocked every other one.self._unpark_tickets: Deque[int] = deque()+ self._unpark_tokens: dict[int, int] = {}+ self._unpark_lapsed: set = set()self._unpark_seq = 0+ self._parked_reclaimable: dict[int, int] = {}+ self._park_seq = 0# KV tokens held by live leases, against the cache size the caller reports. 0 budget disables the check, which# is what every pre-existing caller gets.self._committed = 0@@ -693,8 +749,8 @@ class LlamaAdmissionQueue:self._capacity = capacityself._free = [slot for slot in range(capacity) if not self._in_use >> slot & 1]- def _fits_budget_locked(self, tokens: int) -> bool:- """Whether ``tokens`` more KV may be committed.+ def _fits_against_locked(self, committed: int, tokens: int) -> bool:+ """Whether ``tokens`` fit a cache already holding ``committed``.A caller is always admitted when nothing else holds KV, however large it is. Its request maystill be refused by llama-server, but that refusal names both token counts and the setting@@ -709,14 +765,30 @@ class LlamaAdmissionQueue:"""if self._budget <= 0 or tokens <= 0:return True- if self._committed == 0:+ if committed <= 0:return True- return self._committed + tokens <= self._budget+ return committed + tokens <= self._budget++ def _fits_budget_locked(+ self,+ tokens: int,+ reserved_tokens: int = 0,+ ) -> bool:+ """Whether ``tokens`` more KV fit over the ``reserved_tokens`` owed to resumes."""+ return self._fits_against_locked(self._committed + max(0, reserved_tokens), tokens)++ def _reserved_tokens_locked(self) -> int:+ return sum(+ tokens+ for ticket, tokens in self._unpark_tokens.items()+ if ticket not in self._unpark_lapsed+ )def _can_admit_locked(self,reserved: int,tokens: int = 0,+ reserved_tokens: int = 0,) -> bool:# Slots still held above a shrunk capacity keep occupying the backend, so count every held slot against the# ceiling, not just the ids below it. ``reserved`` holds slots back for approved holders waiting to resume;@@ -725,14 +797,15 @@ class LlamaAdmissionQueue:return False# A free slot is not enough: with --kv-unified every slot reports the full n_ctx, so the pool can hand out more# slots than the one cache can serve.- return self._fits_budget_locked(tokens)+ return self._fits_budget_locked(tokens, reserved_tokens)def _take_slot_locked(self,reserved: int,tokens: int = 0,+ reserved_tokens: int = 0,) -> Optional[int]:- if not self._can_admit_locked(reserved, tokens):+ if not self._can_admit_locked(reserved, tokens, reserved_tokens):return Noneslot = self._free.pop()self._in_use |= 1 << slot@@ -777,7 +850,9 @@ class LlamaAdmissionQueue:# for more, so an arrival admitted here would take exactly that, pinning a growing conversation at its# opening size for as long as traffic lasts.if not self._waiters and self._reparking == 0:- slot = self._take_slot_locked(len(self._unpark_tickets), cost)+ slot = self._take_slot_locked(+ len(self._unpark_tickets), cost, self._reserved_tokens_locked()+ )if slot is not None:# No snapshot here: callers read it through snapshot_now(), which re-reads the queue, so building# one per admitted request would be pure allocation on the hot path.@@ -904,56 +979,144 @@ class LlamaAdmissionQueue:self._committed += max(0, int(restore or 0))self._grant_waiters_locked()- def try_park(self, slot: Optional[int]) -> bool:+ def try_park(+ self,+ slot: Optional[int],+ *,+ tokens: int = 0,+ ) -> Optional[int]:"""Return a parked holder's slot to the pool. See ``LlamaAdmissionLease.park``.- False leaves the slot with its holder, so a refused park costs nothing to- undo. The per-queue count is only what ``is_idle`` reads; the budget and- the capacity it is sized from are both process-wide.+ None leaves the slot with its holder, so a refused park costs nothing to+ undo. Otherwise an id identifying this park until the holder leaves it. The+ per-queue count is only what ``is_idle`` reads; the budget and the capacity it+ is sized from are both process-wide.++ ``tokens`` stays committed, and is recorded only so ``reclaim_would_admit``+ knows how much erasing could free."""if not _claim_park(_max_parked(_live_capacity(self))):- return False+ return Nonewith self._lock:self._parked += 1+ self._park_seq += 1+ park_id = self._park_seq+ self._parked_reclaimable[park_id] = max(0, int(tokens or 0))self._release_slot_locked(slot)self._grant_waiters_locked()- return True+ return park_id- def unpark(self) -> None:+ def reclaim_parked_tokens(+ self,+ tokens: int,+ *,+ park_id: Optional[int] = None,+ ) -> None:+ with self._lock:+ self._parked_reclaimable.pop(park_id, None)+ self._committed = max(0, self._committed - max(0, int(tokens or 0)))+ self._grant_waiters_locked()++ def _ticket_position_locked(self, ticket: int) -> tuple:+ """(slots, room) that resuming holders ahead of ``ticket`` hold back from it.++ One ahead reserves room only until its deadline lapses, but holds a slot back only+ while it can afford that room: pinning one it cannot pay for deadlocks the pool.+ """+ ahead = 0+ ahead_tokens = 0+ for queued in self._unpark_tickets:+ if queued == ticket:+ break+ needs = self._unpark_tokens.get(queued, 0)+ if self._fits_budget_locked(needs, ahead_tokens):+ ahead += 1+ if queued not in self._unpark_lapsed:+ ahead_tokens += needs+ return (ahead, ahead_tokens)++ def _claimants_locked(self):+ for ticket in self._unpark_tickets:+ yield (self._unpark_tokens.get(ticket, 0), *self._ticket_position_locked(ticket))+ self._prune_waiters_locked()+ if self._waiters:+ yield (+ self._waiters[0].tokens,+ len(self._unpark_tickets),+ self._reserved_tokens_locked(),+ )++ def reclaim_would_admit(self, tokens: int) -> bool:+ """Whether erasing parked caches would let anyone admission owes into the cache.++ Weighed in aggregate: one claimant erasing cannot fit must not hide a smaller one,+ nor two too-small holders both decline and leave the queue stalled.+ """+ if max(0, int(tokens or 0)) <= 0:+ return False+ with self._lock:+ if self._budget <= 0 or self._reparking > 0:+ return False+ reclaimable = sum(self._parked_reclaimable.values())+ for wanted, held_back, reserved in self._claimants_locked():+ if self._can_admit_locked(held_back, wanted, reserved):+ continue+ if not (bool(self._free) and (self._held + held_back) < self._capacity):+ continue+ if self._fits_against_locked(+ max(0, self._committed - reclaimable) + reserved, wanted+ ):+ return True+ return False++ def unpark(self, park_id: Optional[int] = None) -> None:with self._lock:if self._parked > 0:self._parked -= 1+ self._parked_reclaimable.pop(park_id, None)async def acquire_parked_slot(self,*,cancel_event = None,poll_s: float = 0.02,+ tokens: int = 0,+ timeout_s: Optional[float] = DEFAULT_RECOST_WAIT_TIMEOUT_S,+ abandoned = None,) -> Optional[int]:- """Wait for a slot for a holder resuming from a park, None if cancelled.+ """Wait for a slot, and for ``tokens`` of cache room, None if cancelled.Ordered by ticket rather than counted, so approvals resume in the orderthey came back: counting them made every approved holder block everyother one, and with nothing decoding that never resolved.++ ``tokens`` is what a holder whose cache was erased must win back, held from arrivals+ until ``timeout_s`` lapses the claim. Lapsing keeps the ticket's place in the slot+ order and still never admits it over budget: llama.cpp answers an exhausted cache by+ clearing every slot it is decoding."""+ tokens = max(0, int(tokens or 0))with self._lock:self._unpark_seq += 1ticket = self._unpark_seqself._unpark_tickets.append(ticket)+ self._unpark_tokens[ticket] = tokens+ deadline = None if not timeout_s or timeout_s <= 0 else time.monotonic() + timeout_stry:while True:with self._lock:- ahead = 0- for queued in self._unpark_tickets:- if queued == ticket:- break- ahead += 1- # Only the approvals ahead of this one hold slots back from it.- slot = self._take_slot_locked(ahead)+ if deadline is not None and time.monotonic() >= deadline:+ self._unpark_lapsed.add(ticket)+ deadline = None+ self._grant_waiters_locked()+ ahead, ahead_tokens = self._ticket_position_locked(ticket)+ slot = self._take_slot_locked(ahead, tokens, ahead_tokens)if slot is not None:return slotif cancel_event is not None and cancel_event.is_set():return None+ if abandoned is not None and abandoned():+ return Noneawait asyncio.sleep(poll_s)finally:with self._lock:@@ -961,6 +1124,8 @@ class LlamaAdmissionQueue:self._unpark_tickets.remove(ticket)except ValueError:pass+ self._unpark_tokens.pop(ticket, None)+ self._unpark_lapsed.discard(ticket)self._grant_waiters_locked()def cancel(self, waiter: _Waiter) -> None:@@ -1007,12 +1172,14 @@ class LlamaAdmissionQueue:# traffic keeps arriving. Head-of-line blocking is the fair trade here, and it matches the FIFO the rest of this# queue already promises.while self._waiters and self._can_admit_locked(- len(self._unpark_tickets), self._waiters[0].tokens+ len(self._unpark_tickets), self._waiters[0].tokens, self._reserved_tokens_locked()):waiter = self._waiters.popleft()if waiter.cancelled or waiter.future.done():continue- slot = self._take_slot_locked(len(self._unpark_tickets), waiter.tokens)+ slot = self._take_slot_locked(+ len(self._unpark_tickets), waiter.tokens, self._reserved_tokens_locked()+ )lease = LlamaAdmissionLease(self, slot, waiter.tokens)waiter.granted_lease = leasetry:diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.pyindex ffc3cfec83d..1f222636590 100644--- a/studio/backend/core/inference/llama_cpp.py+++ b/studio/backend/core/inference/llama_cpp.py@@ -31608,6 +31608,56 @@ class LlamaCppBackend:continueraise+ @staticmethod+ def decode_slot_from_chunk(chunk) -> Optional[int]:+ """The llama-server slot a stream chunk was decoded on, None if it does not say.++ Only the final result carries ``__verbose``, and only when asked for.+ """+ if not isinstance(chunk, dict):+ return None+ verbose = chunk.get("__verbose")+ if not isinstance(verbose, dict):+ return None+ slot = verbose.get("id_slot")+ if type(slot) is not int or slot < 0:+ return None+ return slot++ def release_idle_chat_slot(self, base_url: str, slot: int) -> bool:+ """Erase one completed round's cached context, True once the engine says so.++ Declines unless the round decoded on this server, which a reload renumbers, and+ unless --slot-save-path is in play, which the endpoint requires. An erase aimed at+ a slot since handed to another request is deferred rather than dropped, so even a+ timeout can cost that chat its prefix.+ """+ if (+ not self._slot_save_dir+ or base_url != self.base_url+ or type(slot) is not int+ or slot < 0+ ):+ return False+ try:+ response = httpx.post(+ f"{base_url}/slots/{slot}",+ params = {"action": "erase"},+ headers = self._auth_headers,+ timeout = 2.0,+ trust_env = False,+ )+ response.raise_for_status()+ result = response.json()+ return (+ result.get("id_slot") == slot+ and type(result.get("n_erased")) is int+ and result["n_erased"] >= 0+ )+ except Exception:+ logger.debug("Approval cache reclamation failed; retaining reservation", exc_info = True)+ return False+def generate_chat_completion(self,messages: list[dict],@@ -32056,6 +32106,7 @@ class LlamaCppBackend:# MAY BLOCK: recost_waiting waits for cache room. Safe at the top of a round,# where the previous round's request has completed.on_conversation_grew: Optional[Callable[[list], None]] = None,+ on_decode_slot: Optional[Callable[[str, int], None]] = None,) -> Generator[dict, None, None]:"""Agentic loop: let the model call tools, execute them, and continue.@@ -32774,6 +32825,9 @@ class LlamaCppBackend:# Progress events feed the first-token deadline; timings stay opt-in.payload["return_progress"] = True+ if on_decode_slot is not None:+ payload["verbose"] = True+ payload["response_fields"] = ["id_slot"]if perf_callback is not None:payload["timings_per_token"] = Trueif logit_bias:@@ -33057,6 +33111,10 @@ class LlamaCppBackend:try:chunk_data = json.loads(line[6:])+ if on_decode_slot is not None:+ slot = self.decode_slot_from_chunk(chunk_data)+ if slot is not None:+ on_decode_slot(self.base_url, slot)_report_live_llama_timings(perf_callback, chunk_data)_ct = chunk_data.get("timings")diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.pyindex 03dea9f0f3c..c8ad552757f 100644--- a/studio/backend/routes/inference.py+++ b/studio/backend/routes/inference.py@@ -1748,6 +1748,7 @@ _TEARDOWN_TASK_STOP_TIMEOUT_S = 5.0# (e.g. prompt prefill between tool iterations). A second layer atop the# tool_stream_exec heartbeats, keeping proxies (Cloudflare drops idle at ~100s)._LOCAL_TOOL_STREAM_STALL_KEEPALIVE_S = 15.0+_APPROVAL_CACHE_RECLAIM_POLL_S = 0.25def _openai_llama_admission_capacity(request: Optional[Request], llama_backend = None) -> int:@@ -2225,6 +2226,7 @@ def _openai_llama_admission_recost(output_tokens: Optional[int] = None,cancel_event = None,injected_tools = None,+ cache_is_empty: bool = False,) -> None:"""Charge a tool loop for what its conversation now is, not what it opened as.@@ -2241,6 +2243,10 @@ def _openai_llama_admission_recost(Safe to wait here because it is between rounds and the slot is idle at llama-server.Idle is not reclaimed, though, so the yield is gated on``_openai_llama_admission_can_yield``; where it is False this declines instead.++ ``cache_is_empty`` overrides that gate for a round whose cells were explicitly erased.+ The gate exists because an idle slot's KV stays resident, which an erased slot's does+ not, so yielding there hands back room that really is free."""if reservation is None:return@@ -2287,7 +2293,7 @@ def _openai_llama_admission_recost(lease.recost_waiting(want,cancel_event = cancel_event,- allow_yield = _openai_llama_admission_can_yield(llama_backend),+ allow_yield = cache_is_empty or _openai_llama_admission_can_yield(llama_backend),)except Exception: # pragma: no cover - accounting must not break a live runlogger.debug("llama admission recost failed", exc_info = True)@@ -23459,8 +23465,19 @@ async def produce_openai_chat_completions(# reservation exists but not ITERATED until after, so the callback always sees# a reservation by the time a round can call it._gguf_admission_hold: dict = {"reservation": None}+ _gguf_decode_lock = threading.Lock()+ _gguf_decode: dict = {"slot": None, "erased": False}++ def _gguf_record_decode_slot(base_url: str, slot: int) -> None:+ with _gguf_decode_lock:+ _gguf_decode["slot"] = (base_url, slot)+ _gguf_decode["erased"] = Falsedef _gguf_recost(conversation) -> None:+ with _gguf_decode_lock:+ erased = _gguf_decode["erased"]+ _gguf_decode["slot"] = None+ _gguf_decode["erased"] = False_openai_llama_admission_recost(_gguf_admission_hold["reservation"],conversation,@@ -23477,6 +23494,8 @@ async def produce_openai_chat_completions(# A round waiting for cache room must still answer Stop. Same event# the loop polls each iteration, so a wait ends where a cancel would.cancel_event = cancel_event,+ # Erased cells make yielding honest, so a grown round may wait.+ cache_is_empty = erased,)# Active tool names gating the bare-rehearsal strip, matching the loop gate.@@ -23542,6 +23561,10 @@ async def produce_openai_chat_completions(permission_mode = payload.permission_mode,perf_callback = _gguf_perf_callback,on_conversation_grew = _gguf_recost,+ # Only the streaming path parks and reclaims, so only it can use a slot.+ on_decode_slot = _gguf_record_decode_slot+ if payload.stream and _effective_confirm and not payload.bypass_permissions+ else None,context_overflow = _rolling_context_policy(payload),context_policy = _request_context_policy(payload),compaction_headroom_ratio = _request_compaction_headroom_ratio(payload),@@ -23594,6 +23617,77 @@ async def produce_openai_chat_completions(# otherwise unanswered prompts hold every slot._parked = False+ _reclaim_task = None+ _reclaim_stop = asyncio.Event()++ def _erase_and_record(target) -> bool:+ if not llama_backend.release_idle_chat_slot(*target):+ return False+ with _gguf_decode_lock:+ _gguf_decode["erased"] = True+ return True++ async def _reclaim_approval_cache(lease):+ """Erase this round's cached context once a queued chat needs its room.++ On demand only: the engine drops the cells rather than spilling them,+ so this costs the approved chat a full reprocess. Stopping is honoured+ only before the erase is sent; past that the cells are going regardless.+ """+ try:+ while not lease.reclaim_would_admit():+ try:+ await asyncio.wait_for(+ _reclaim_stop.wait(), _APPROVAL_CACHE_RECLAIM_POLL_S+ )+ except asyncio.TimeoutError:+ continue+ return+ with _gguf_decode_lock:+ target = _gguf_decode["slot"]+ erased = _gguf_decode["erased"]+ if target is None or _reclaim_stop.is_set():+ return+ if not erased and not await asyncio.to_thread(_erase_and_record, target):+ return+ lease.release_parked_cache()+ except asyncio.CancelledError:+ raise+ except Exception:+ logger.debug("Approval cache reclamation failed", exc_info = True)++ def _watch_for_reclaim(lease) -> None:+ nonlocal _reclaim_task+ with _gguf_decode_lock:+ known = _gguf_decode["slot"] is not None+ if not known or _reclaim_task is not None:+ return+ _reclaim_stop.clear()+ _reclaim_task = asyncio.create_task(_reclaim_approval_cache(lease))++ async def _settle_reclaim_watch() -> None:+ """Wait out a reclamation that is already sending, before resuming.++ Cancelling would not stop it: the worker runs on and the erase lands+ anyway, leaving the resumed round priced against cells mid-drop.+ """+ nonlocal _reclaim_task+ task, _reclaim_task = _reclaim_task, None+ if task is None:+ return+ _reclaim_stop.set()+ try:+ await task+ except Exception:+ logger.debug("Approval cache reclamation failed", exc_info = True)++ def _stop_reclaim_watch() -> None:+ nonlocal _reclaim_task+ _reclaim_stop.set()+ if _reclaim_task is not None:+ _reclaim_task.cancel()+ _reclaim_task = None+async def _park_admission(on: bool, *, wait: bool = True):nonlocal _parkedif on == _parked:@@ -23604,16 +23698,15 @@ async def produce_openai_chat_completions(if lease is None:returnif on:- # Refused when the budget is spent: the slot stays here,- # so there is nothing to take back afterwards.if not lease.park():return+ _watch_for_reclaim(lease)elif wait:- # Resuming: park() may have handed our slot to a waiter, so wait for room instead- # of putting two holders on one slot.+ await _settle_reclaim_watch()await lease.unpark_async(cancel_event = cancel_event)else:# Tearing down; the lease is released separately.+ _stop_reclaim_watch()lease.unpark()_parked = ondiff --git a/studio/backend/tests/test_llama_admission_compat_matrix.py b/studio/backend/tests/test_llama_admission_compat_matrix.pyindex 77155641382..b4b4688523d 100644--- a/studio/backend/tests/test_llama_admission_compat_matrix.py+++ b/studio/backend/tests/test_llama_admission_compat_matrix.py@@ -184,6 +184,10 @@ class TestNothingChangesWhenTheBudgetIsUnknown:assert queue._reparking == 0+# In the order they were added, so older positional callers keep their meaning.+_TOOL_LOOP_HOOKS = ("on_conversation_grew", "on_decode_slot")++class TestOldCallers:"""Everything added is keyword-with-default, so code written before this still runs."""@@ -220,8 +224,8 @@ class TestOldCallers:from core.inference.llama_cpp import LlamaCppBackendsignature = inspect.signature(LlamaCppBackend.generate_chat_completion_with_tools)- parameter = signature.parameters["on_conversation_grew"]- assert parameter.default is None, "the hook must be optional for existing callers"+ for name in _TOOL_LOOP_HOOKS:+ assert signature.parameters[name].default is None, f"{name} must be optional"def test_the_hook_was_appended_rather_than_inserted(self):"""No bare ``*`` in this signature, so every parameter is positional-or-keyword and@@ -234,9 +238,8 @@ class TestOldCallers:names = list(inspect.signature(LlamaCppBackend.generate_chat_completion_with_tools).parameters)- assert (- names[-1] == "on_conversation_grew"- ), f"the hook must be last; signature ends {names[-3:]}"+ tail = names[-len(_TOOL_LOOP_HOOKS) :]+ assert tail == list(_TOOL_LOOP_HOOKS), f"the hooks must stay at the tail, got {tail}"def test_the_wait_timeout_has_a_sane_default(self):import inspect@@ -245,8 +248,9 @@ class TestOldCallers:from core.inference.llama_admission import LlamaAdmissionLease- signature = inspect.signature(LlamaAdmissionLease.recost_waiting)- assert signature.parameters["timeout_s"].default == DEFAULT_RECOST_WAIT_TIMEOUT_S+ for method in (LlamaAdmissionLease.recost_waiting, LlamaAdmissionLease.unpark_async):+ signature = inspect.signature(method)+ assert signature.parameters["timeout_s"].default == DEFAULT_RECOST_WAIT_TIMEOUT_Sclass TestNoPersistentStateChanged:diff --git a/studio/backend/tests/test_llama_cpp_tool_loop.py b/studio/backend/tests/test_llama_cpp_tool_loop.pyindex 5ce59f06af4..43884f5f758 100644--- a/studio/backend/tests/test_llama_cpp_tool_loop.py+++ b/studio/backend/tests/test_llama_cpp_tool_loop.py@@ -6304,6 +6304,27 @@ def test_the_synthesized_final_pass_is_recosted_before_it_is_sent(monkeypatch):)[email protected]("wanted", [True, False])+def test_the_decode_slot_is_asked_for_and_read_only_when_a_caller_wants_it(monkeypatch, wanted):+ """Verbose alone would attach the whole prompt, so it travels with the narrowing."""+ final = {"choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}]}+ final["__verbose"] = {"id_slot": 3}+ stream = [_sse({"content": "done"}), "data: " + json.dumps(final) + "\n", _done()]+ backend, payloads = _backend_and_payloads(monkeypatch, [stream])+ seen = []++ _run_tool_loop(+ backend,+ [{"role": "user", "content": "hi"}],+ _render_html_tools(),+ on_decode_slot = (lambda url, slot: seen.append((url, slot))) if wanted else None,+ )++ assert seen == ([(backend.base_url, 3)] if wanted else [])+ assert payloads[0].get("verbose") is (True if wanted else None)+ assert payloads[0].get("response_fields") == (["id_slot"] if wanted else None)++@pytest.mark.parametrize("held",[diff --git a/studio/backend/tests/test_tool_approval_context_release.py b/studio/backend/tests/test_tool_approval_context_release.pynew file mode 100644index 00000000000..6eeedab6fe0--- /dev/null+++ b/studio/backend/tests/test_tool_approval_context_release.py@@ -0,0 +1,115 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++"""A chat parked on a tool approval gives its cached context back on demand."""++import asyncio+from types import SimpleNamespace++import pytest+import core.inference.llama_cpp as llama_cpp+from core.inference.llama_admission import (+ LlamaAdmissionConfig,+ LlamaAdmissionQueue,+ reset_llama_admission_queues,+)++[email protected](autouse = True)+def reset():+ reset_llama_admission_queues()+ yield+ reset_llama_admission_queues()+++def reserve(queue, tokens, capacity):+ config = LlamaAdmissionConfig()+ return queue.reserve(capacity = capacity, config = config, tokens = tokens, budget = 35000)+++def test_a_parked_holder_keeps_its_context_until_someone_is_blocked_on_it():+ async def scenario():+ queue = LlamaAdmissionQueue("demand")+ parked = reserve(queue, 29000, 4).lease_nowait()+ assert parked.park()+ assert queue.snapshot().committed == 29000, "parking returns the slot, not the KV"+ assert not parked.reclaim_would_admit(), "erasing costs a reprocess, admits nobody"+ waiter = reserve(queue, 8750, 4)+ assert waiter.lease_nowait() is None+ assert parked.reclaim_would_admit()+ waiter.cancel()+ assert not parked.reclaim_would_admit()++ asyncio.run(scenario())+++def test_a_resume_wins_its_room_back_and_is_not_overtaken():+ async def scenario():+ queue = LlamaAdmissionQueue("resume")+ first = reserve(queue, 29000, 4).lease_nowait()+ assert first.park()+ assert first.release_parked_cache() == 29000+ busy = reserve(queue, 20000, 4).lease_nowait() # blocks the resume, not a small arrival+ resumed = asyncio.create_task(first.unpark_async(poll_s = 0.001))+ await asyncio.sleep(0.01)+ assert not resumed.done(), "a spare slot does not justify exceeding the KV budget"+ arrivals = [reserve(queue, 8000, 4) for _ in range(4)]+ assert not any(a.lease_nowait() for a in arrivals), "the approved chat is owed this"+ busy.release()+ await asyncio.wait_for(resumed, 1)+ assert queue.snapshot().committed == 29000, "the queue drains to the resumer first"+ for arrival in arrivals:+ arrival.cancel()+ first.release()+ assert queue.snapshot().committed == 0, "what the resume took back is given back"++ asyncio.run(scenario())++[email protected]("timeout_s", [None, 0.01])+def test_a_claim_it_cannot_pay_for_does_not_pin_the_slot_a_later_one_needs(timeout_s):+ """The first needs KV the second holds; the second needs the slot the first reserves."""++ async def scenario():+ queue = LlamaAdmissionQueue(f"deadlock-{timeout_s}")+ first = reserve(queue, 10000, 1).lease_nowait()+ assert first.park()+ assert first.release_parked_cache() == 10000+ second = reserve(queue, 35000, 1).lease_nowait()+ assert second is not None, "the whole cache, freed by the first chat's erasure"+ assert second.park()+ ahead = asyncio.create_task(first.unpark_async(poll_s = 0.001, timeout_s = timeout_s))+ await asyncio.sleep(0.05)+ assert not ahead.done(), "the cache is committed; 10000 more does not fit"+ await asyncio.wait_for(second.unpark_async(poll_s = 0.001), 1)+ second.release()+ await asyncio.wait_for(ahead, 1)+ assert queue.snapshot().committed == 10000++ asyncio.run(scenario())+++ "save_dir,other,body,ok",+ [+ ("/tmp/slots", False, {"id_slot": 2, "n_erased": 12}, True),+ ("/tmp/slots", False, {"id_slot": 1, "n_erased": 12}, False),+ ("/tmp/slots", False, {"id_slot": 2}, False),+ ("/tmp/slots", True, None, False),+ (None, False, None, False),+ ],+)+def test_erasure_unconfirmed_keeps_the_context(monkeypatch, save_dir, other, body, ok):+ server = SimpleNamespace(base_url = "http://s", _auth_headers = {}, _slot_save_dir = save_dir)++ sent = []++ def post(url, **kwargs):+ sent.append((url, kwargs["params"]))+ return SimpleNamespace(raise_for_status = lambda: None, json = lambda: body)++ monkeypatch.setattr(llama_cpp.httpx, "post", post)+ target = "http://other" if other else server.base_url+ assert llama_cpp.LlamaCppBackend.release_idle_chat_slot(server, target, 2) is ok+ # A swallowed request reads as a refusal, so the call itself is the assertion.+ assert sent == ([] if body is None else [("http://s/slots/2", {"action": "erase"})])