Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: cancel + bound conversation-archive embeddings before inference (#14519) (#14546)
Indrajit swain7dbc8b9ec890
diff --git a/studio/backend/core/inference/diffusion.py b/studio/backend/core/inference/diffusion.pyindex f05a623562a..27bdfc9996b 100644--- a/studio/backend/core/inference/diffusion.py+++ b/studio/backend/core/inference/diffusion.py@@ -35,6 +35,7 @@ from dataclasses import dataclass, field, replacefrom pathlib import Pathfrom typing import Any, Callable, Mapping, Optional, Sequence+from core.inference.inference_gate import holds_inference, pauses_background_embeddingfrom core._torchao_stub import (alias_torchao_fqn_to_config,hide_xformers_built_for_another_torch,@@ -4722,6 +4723,7 @@ class DiffusionBackend:assert_pick_is_not_speech(repo_id, gguf_filename, hf_token, allow_network)@_account_owned_load+ @pauses_background_embeddingdef begin_load(self,repo_id: str,@@ -12582,6 +12584,7 @@ class DiffusionBackend:)@_release_render_on_unload+ @holds_inference@with_hiresdef generate(self,diff --git a/studio/backend/core/inference/inference_gate.py b/studio/backend/core/inference/inference_gate.pynew file mode 100644index 00000000000..cc5fdff0c64--- /dev/null+++ b/studio/backend/core/inference/inference_gate.py@@ -0,0 +1,100 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""Mark a function as inference so background archive embedding never runs alongside it."""++from __future__ import annotations++import functools+import inspect+import weakref++import httpx+++def holds_inference(fn):+ """The archive's gap worker pauses (waiting out its batch in flight) for ``fn``'s whole run; a generator+ holds it until exhausted or closed. Imported lazily: this module must stay free of the RAG stack."""+ if inspect.iscoroutinefunction(fn):++ @functools.wraps(fn)+ async def coroutine(*args, **kwargs):+ import asyncio++ from core.rag.conversation_archive import _pause_prewarm, hold_inference++ release = hold_inference()+ try:+ # The join waits out the batch in flight, so off the event loop.+ await asyncio.to_thread(_pause_prewarm)+ return await fn(*args, **kwargs)+ finally:+ release()++ return coroutine++ if inspect.isgeneratorfunction(fn):++ @functools.wraps(fn)+ def generator(*args, **kwargs):+ from core.rag.conversation_archive import inference_running+ with inference_running():+ yield from fn(*args, **kwargs)++ return generator++ @functools.wraps(fn)+ def call(*args, **kwargs):+ from core.rag.conversation_archive import inference_running+ with inference_running():+ return fn(*args, **kwargs)++ return call+++def pauses_background_embedding(fn):+ """For starts of long jobs (training): stop the gap worker and wait out its batch before admitting the job;+ the worker's own busy probe keeps it idle for the job's lifetime."""++ @functools.wraps(fn)+ def call(*args, **kwargs):+ from core.rag.conversation_archive import _busy++ # Held across the start, so a stream ending meanwhile cannot restart the worker before the job publishes+ # the state the busy probe reads.+ with _busy(restart = False):+ return fn(*args, **kwargs)++ return call+++def gated_async_client(*args, **kwargs):+ """A per-request ``httpx.AsyncClient`` to a local engine (API passthroughs): background embedding stays paused+ until it is ``aclose``d, or collected if a path never closes it. Built through ``httpx.AsyncClient`` at call+ time, so a patched client class is honoured."""+ from core.rag.conversation_archive import hold_inference++ client = httpx.AsyncClient(*args, **kwargs)+ release = weakref.finalize(client, hold_inference())+ close = client.aclose+ send = getattr(client, "send", None)++ async def gated_send(*send_args, **send_kwargs):+ # The first request waits out the batch in flight (off the event loop); httpx routes every verb and+ # stream() through send.+ from core.rag.conversation_archive import _pause_prewarm, prewarm_running+ if prewarm_running():+ import asyncio+ await asyncio.to_thread(_pause_prewarm)+ return await send(*send_args, **send_kwargs)++ async def aclose():+ try:+ return await close()+ finally:+ release()++ client.aclose = aclose+ if inspect.iscoroutinefunction(send):+ client.send = gated_send+ return clientdiff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.pyindex d5f9942330f..fd78923ff4f 100644--- a/studio/backend/core/inference/llama_cpp.py+++ b/studio/backend/core/inference/llama_cpp.py@@ -22,6 +22,7 @@ import mathimport osimport reimport struct+from core.inference.inference_gate import holds_inferencefrom loggers import get_loggerfrom utils.gpu_memory_events import invalidate_gpu_memory as _invalidate_gpu_memoryfrom utils import host_memory as _host_memory@@ -2289,6 +2290,8 @@ def _archive_and_recall(recall_budget_tokens: int = 0,count_tokens: Optional[Callable[[list[dict]], int]] = None,branch_messages: Optional[list[dict]] = None,+ cancel_check: Optional[Callable[[], bool]] = None,+ background: bool = False,) -> dict:"""Keep the turns this fit evicted, then pull the relevant ones back.@@ -2313,19 +2316,26 @@ def _archive_and_recall(return resultgone = evicted_messages(before, conversation)+ if background:+ # llama.cpp only: its streams gate the idle-gap worker (`inference_running`).+ conversation_archive.note_branch(thread_id, before, cancel_check)# `conversation` is the fitted prompt, so it says which repeats of a turn the model# can still read. Without it a twice-said turn is archived twice and spends two# recall slots on text already in the prompt.archived = (conversation_archive.archive_turns(- thread_id, gone, live = conversation, branch = branch_messages or conversation+ thread_id,+ gone,+ live = conversation,+ branch = branch_messages or conversation,+ cancel_check = cancel_check,)if goneelse 0)counts = {"archived_messages": archived}- if recall_done:+ if recall_done or (cancel_check is not None and cancel_check()):result["counts"] = countsreturn resultif not force_recall:@@ -25938,6 +25948,7 @@ class LlamaCppBackend:@_invalidates_gpu_memory("llama.cpp load")@_with_gguf_load_marker+ @holds_inferencedef load_model(self,intent: GgufLoadIntent,@@ -39325,13 +39336,18 @@ class LlamaCppBackend:or getattr(self, "_chat_template", None),),}+ from core.rag.conversation_archive import inference_running+stream_timeout = httpx.Timeout(connect = 10, read = 0.5, write = 10, pool = 10)- with httpx.Client(- timeout = stream_timeout,- limits = httpx.Limits(max_keepalive_connections = 0),- trust_env = False,- verify = _local_ssl_context(),- ) as client:+ with (+ inference_running(),+ httpx.Client(+ timeout = stream_timeout,+ limits = httpx.Limits(max_keepalive_connections = 0),+ trust_env = False,+ verify = _local_ssl_context(),+ ) as client,+ ):first_token_deadline = start_first_token_wait()with self._stream_with_retry(client,@@ -40065,6 +40081,8 @@ class LlamaCppBackend:chat_template_kwargs = _reasoning_kw,continue_final_message = continue_final_message,),+ cancel_check = lambda: bool(cancel_event and cancel_event.is_set()),+ background = True,)openai_messages = _recalled["conversation"]truncation = {**truncation, **_recalled["counts"]}@@ -41139,6 +41157,8 @@ class LlamaCppBackend:strict = True,chat_template_kwargs = _reasoning_kw,),+ cancel_check = lambda: bool(cancel_event and cancel_event.is_set()),+ background = True,)conversation = _recalled["conversation"]truncation = {**truncation, **_recalled["counts"]}@@ -41300,6 +41320,8 @@ class LlamaCppBackend:style = "inline",recall_done = True,branch_messages = _extend_live_branch(conversation),+ cancel_check = lambda: bool(cancel_event and cancel_event.is_set()),+ background = True,)["counts"])payload["messages"] = neutralize_control_markup_in_messages(@@ -43990,6 +44012,8 @@ class LlamaCppBackend:strict = True,chat_template_kwargs = _reasoning_kw,),+ cancel_check = lambda: bool(cancel_event and cancel_event.is_set()),+ background = True,)conversation = _recalled["conversation"]truncation = {**truncation, **_recalled["counts"]}@@ -44134,6 +44158,8 @@ class LlamaCppBackend:style = "inline",recall_done = True,branch_messages = _extend_live_branch(conversation),+ cancel_check = lambda: bool(cancel_event and cancel_event.is_set()),+ background = True,)["counts"])# Recorded rather than only forwarded, for the same reason as the@@ -45120,6 +45146,7 @@ class LlamaCppBackend:LlamaCppBackend._codec_mgr.load_codec(audio_type, device, model_repo_path = model_repo_path)logger.info(f"Loaded audio codec for GGUF TTS: {audio_type}")+ @holds_inferencedef generate_audio_response(self,text: str,diff --git a/studio/backend/core/inference/mlx_diffusion_backend.py b/studio/backend/core/inference/mlx_diffusion_backend.pyindex 9a6f3fb9b7b..d398c061998 100644--- a/studio/backend/core/inference/mlx_diffusion_backend.py+++ b/studio/backend/core/inference/mlx_diffusion_backend.py@@ -20,6 +20,7 @@ from dataclasses import dataclass, fieldfrom pathlib import Pathfrom typing import Any, Callable, Optional, Union+from core.inference.inference_gate import holds_inference, pauses_background_embeddingfrom core.inference.diffusion_families import (DIFFUSION_CANCELLED_MSG,DIFFUSION_NOT_LOADED_MSG,@@ -880,6 +881,7 @@ class MlxDiffusionBackend:# ── Load ─────────────────────────────────────────────────────────────────+ @pauses_background_embeddingdef begin_load(self,repo_id: str,@@ -1235,6 +1237,7 @@ class MlxDiffusionBackend:# ── Generate ─────────────────────────────────────────────────────────────+ @holds_inferencedef generate(self,*,diff --git a/studio/backend/core/inference/orchestrator.py b/studio/backend/core/inference/orchestrator.pyindex ea8f2bae513..92fe0c0edab 100644--- a/studio/backend/core/inference/orchestrator.py+++ b/studio/backend/core/inference/orchestrator.py@@ -14,6 +14,7 @@ import contextlibimport osimport signalfrom loggers import get_logger+from core.inference.inference_gate import holds_inferencefrom utils.gpu_memory_events import invalidates_gpu_memory as _invalidates_gpu_memoryimport multiprocessing as mpimport queue@@ -1743,6 +1744,7 @@ class InferenceOrchestrator:logger.exception("Inference dispatcher: failed to route a response; continuing")continue+ @holds_inferencedef _generate_dispatched(self,messages: list = None,@@ -2068,6 +2070,7 @@ class InferenceOrchestrator:load_generation: int = 0@_invalidates_gpu_memory("inference load")+ @holds_inferencedef load_model(self,config,@@ -2675,6 +2678,7 @@ class InferenceOrchestrator:# Dictation models run in the STT sidecars (whisper-server, llama-server, and the Transformers spawn child), not# the chat worker. Their lifecycle goes through here all the same, so one object knows everything that is resident# and Voice settings and Model Hub cannot report different things about one model.+ @holds_inferencedef load_stt_model(self,model: Optional[str],@@ -3041,6 +3045,7 @@ class InferenceOrchestrator:reply_returns = calls_tools,),count_tokens = _count,+ cancel_check = lambda: bool(cancel_event is not None and cancel_event.is_set()),)fitted = recall["conversation"]truncation = {**truncation, **recall["counts"]}@@ -3592,6 +3597,7 @@ class InferenceOrchestrator:if callable(close):close()+ @holds_inferencedef _generate_inner(self,messages: list = None,@@ -3894,6 +3900,7 @@ class InferenceOrchestrator:)return outputs+ @holds_inferencedef generate_audio_response(self,text: str,@@ -4145,6 +4152,7 @@ class InferenceOrchestrator:self._release_worker(cancel_event)release_mailbox()+ @holds_inferencedef generate_whisper_response(self,audio_array,@@ -4165,6 +4173,7 @@ class InferenceOrchestrator:extra_audio_arrays = extra_audio_arrays,)+ @holds_inferencedef generate_audio_input_response(self,messages,diff --git a/studio/backend/core/inference/sd_cpp_backend.py b/studio/backend/core/inference/sd_cpp_backend.pyindex f7a56ef94d1..742c3611ad6 100644--- a/studio/backend/core/inference/sd_cpp_backend.py+++ b/studio/backend/core/inference/sd_cpp_backend.py@@ -34,6 +34,7 @@ from dataclasses import dataclass, field, replacefrom pathlib import Pathfrom typing import Any, Optional+from core.inference.inference_gate import holds_inference, pauses_background_embeddingfrom core.inference.diffusion_auto_policy import build_resolved_record, format_generation_for_logfrom core.inference.diffusion_compat import flux2_inner_dim_for_pickfrom core.inference.diffusion_device import (@@ -3257,6 +3258,7 @@ class SdCppDiffusionBackend:)return upgraded+ @pauses_background_embeddingdef begin_load(self,repo_id: str,@@ -4423,6 +4425,7 @@ class SdCppDiffusionBackend:and workflow in (None, "reference"))+ @holds_inferencedef generate(self,*,diff --git a/studio/backend/core/inference/stt_registry.py b/studio/backend/core/inference/stt_registry.pyindex 37932dedd8f..3f43312b097 100644--- a/studio/backend/core/inference/stt_registry.py+++ b/studio/backend/core/inference/stt_registry.py@@ -15,6 +15,7 @@ from __future__ import annotationsimport threadingfrom typing import Any, Optional, Sequence+from core.inference.inference_gate import holds_inferencefrom loggers import get_loggerfrom utils.gpu_memory_events import invalidates_gpu_memory as _invalidates_gpu_memory@@ -45,6 +46,7 @@ def sidecar_for(engine: str) -> Any:@_invalidates_gpu_memory("stt load")+@holds_inferencedef load(model: Optional[str],engine: str,diff --git a/studio/backend/core/inference/video.py b/studio/backend/core/inference/video.pyindex 81302a7e8c5..9754b54a24c 100644--- a/studio/backend/core/inference/video.py+++ b/studio/backend/core/inference/video.py@@ -296,6 +296,8 @@ from .video_nvenc import encode_nvenc, nvenc_gpufrom .video_stream_encode import claim_stream, offer_stream, open_stream_encodefrom utils.hardware import clear_gpu_cache+from core.inference.inference_gate import holds_inference, pauses_background_embedding+# Shared with the image backend so both pin every loader call to the same live cache rootfrom core.inference.diffusion import hub_cache_dir@@ -4384,6 +4386,7 @@ class VideoBackend:# ── background load + progress ───────────────────────────────────────────+ @pauses_background_embeddingdef begin_load(self,repo_id: str,@@ -11598,6 +11601,7 @@ class VideoBackend:video_id = (video or {}).get("id"),)+ @holds_inferencedef generate(self,*,diff --git a/studio/backend/core/rag/conversation_archive.py b/studio/backend/core/rag/conversation_archive.pyindex 6ee7ee4dfd7..caceee0b3c3 100644--- a/studio/backend/core/rag/conversation_archive.py+++ b/studio/backend/core/rag/conversation_archive.py@@ -20,8 +20,13 @@ import hashlibimport loggingimport bisectimport json+import osimport re-from typing import Optional+import threading+import time+from collections import OrderedDict+from contextlib import contextmanager+from typing import Callable, Optionalfrom storage import rag_db@@ -31,6 +36,17 @@ from .parsers import Pagelogger = logging.getLogger(__name__)+_ARCHIVE_EMBED_BATCH = 64+# Per call; the rest is archived by later requests, which re-evict the same turns.+_ARCHIVE_EMBED_MAX_CHUNKS_DEFAULT = 256+# Background pre-embedding runs only while no inference runs (tool calls, between turns); small batches so+# a new generation waits at most one of them.+_PREWARM_BATCH = 16+_PREWARM_MAX_CHUNKS = 2048+_VECTOR_CACHE_MAX = 8192+_NOTED_MAX = 8+_PAUSE_JOIN_S = 10.0+# Instructions, not conversation. An archived system prompt could come back quoted as an "earlier turn" and read as# user-authored text._SKIP_ROLES = frozenset({"system", "developer"})@@ -358,11 +374,389 @@ def can_archive(thread_id: Optional[str]) -> bool:return False+def _archive_embed_max_chunks() -> int:+ raw = os.environ.get("RAG_ARCHIVE_EMBED_MAX_CHUNKS", "").strip()+ if not raw:+ return _ARCHIVE_EMBED_MAX_CHUNKS_DEFAULT+ try:+ return max(1, int(raw))+ except ValueError:+ return _ARCHIVE_EMBED_MAX_CHUNKS_DEFAULT+++def _chunk_key(identity: str, text: str) -> tuple[str, str]:+ return identity, hashlib.sha256(text.encode("utf-8", "ignore")).hexdigest()+++_vector_cache: "OrderedDict[tuple[str, str], object]" = OrderedDict()+_vector_cache_lock = threading.Lock()+++def _cached_vectors(identity: Optional[str], texts: list[str]) -> list:+ """The cached vector per text, None for a miss. Keyed by content, so an edited turn simply misses."""+ if not identity:+ return [None] * len(texts)+ with _vector_cache_lock:+ out = []+ for text in texts:+ key = _chunk_key(identity, text)+ vector = _vector_cache.get(key)+ if vector is not None:+ _vector_cache.move_to_end(key)+ out.append(vector)+ return out+++def _remember_vectors(identity: str, texts: list[str], vectors) -> None:+ with _vector_cache_lock:+ for text, vector in zip(texts, vectors):+ key = _chunk_key(identity, text)+ # A row of an (N, dim) batch is a view: cached as is, it would keep the whole batch alive.+ copy = getattr(vector, "copy", None)+ _vector_cache[key] = copy() if copy is not None else vector+ _vector_cache.move_to_end(key)+ while len(_vector_cache) > _VECTOR_CACHE_MAX:+ _vector_cache.popitem(last = False)+++def _turn_chunks(text: str, chunk_limit: int, count) -> list:+ """The exact chunking archive_turns stores, shared so pre-embedded vectors match it."""+ return chunk_pages(+ [Page(text = text, page_number = None, char_count = len(text))],+ max_tokens = chunk_limit,+ overlap = config.CHUNK_OVERLAP,+ count = count,+ )+++def _embed_pending_chunks(+ texts: list[str],+ *,+ model_name: str,+ cancel_check: Optional[Callable[[], bool]] = None,+ expected_identity: Optional[str] = None,+ cached: Optional[list] = None,+ max_encode: Optional[int] = None,+):+ """``(vectors, identity, cancelled, error)``, partial after a cancel or a late failure:+ write only groups ``vectors`` fully covers. A failing first batch raises. Only cache misses+ are encoded."""+ # `cached` is the lookup the caller's cap was computed from; a second lookup could see a later eviction.+ vectors = list(cached) if cached is not None else _cached_vectors(expected_identity, texts)+ identity = expected_identity if any(v is not None for v in vectors) else None+ misses = [i for i, v in enumerate(vectors) if v is None]+ restarted = cancelled = False+ error = None+ k = 0+ while k < len(misses):+ if cancel_check is not None and cancel_check():+ cancelled = True+ break+ idx = misses[k : k + _ARCHIVE_EMBED_BATCH]+ try:+ out, batch_identity = embeddings.encode_with_identity(+ [texts[i] for i in idx], model_name = model_name, normalize = True+ )+ except Exception as exc:+ # Keep finished batches, or a late timeout makes every request redo the whole pass.+ error = exc+ break+ if identity is not None and batch_identity != identity:+ if restarted:+ break+ # An ST failure swaps to llama-server for good: restart in the new space, as ingestion does.+ logger.warning("conversation_archive.embedder_changed_mid_pass; re-embedding")+ vectors, identity, restarted, k = [None] * len(texts), None, True, 0+ # Former cache hits are re-encoded too, so the restart honours the request's cap itself.+ misses = list(range(len(texts) if max_encode is None else min(len(texts), max_encode)))+ continue+ identity = batch_identity+ for i, vector in zip(idx, out):+ vectors[i] = vector+ _remember_vectors(batch_identity, [texts[i] for i in idx], out)+ k += _ARCHIVE_EMBED_BATCH+ prefix = []+ for vector in vectors:+ if vector is None:+ break+ prefix.append(vector)+ if error is not None and not prefix:+ raise error+ return prefix, identity or embeddings.embedding_identity(model_name), cancelled, error+++_gap = threading.Lock()+_inference_depth = 0+_noted_branches: dict = {}+_prewarm_worker: Optional[threading.Thread] = None+_prewarm_stop = threading.Event()+++def note_branch(+ thread_id: Optional[str],+ messages: list[dict],+ cancel_check: Optional[Callable[[], bool]] = None,+) -> None:+ """Remember a thread's latest transcript for the next idle gap; replaces the previous snapshot."""+ if not thread_id or not messages or not enabled():+ return+ from utils.account_context import current_account++ # Captured here: the worker is a plain thread, which would otherwise read the owner's databases.+ account = current_account()+ with _gap:+ # A stopped request whose stream never opened would otherwise keep its transcript here indefinitely.+ for key, job in list(_noted_branches.items()):+ if job[3] is not None and job[3]():+ del _noted_branches[key]+ key = (account.account_id, thread_id)+ _noted_branches.pop(key, None)+ while len(_noted_branches) >= _NOTED_MAX:+ del _noted_branches[next(iter(_noted_branches))]+ _noted_branches[key] = (+ account,+ thread_id,+ list(messages),+ cancel_check,+ )+++@contextmanager+def _busy(restart: bool):+ global _inference_depth+ with _gap:+ _inference_depth += 1+ _pause_prewarm()+ try:+ yield+ finally:+ with _gap:+ _inference_depth -= 1+ if restart:+ _start_prewarm()+++def inference_running():+ """Hold while llama.cpp generates. Stops background embedding and waits out its batch in flight."""+ return _busy(restart = True)+++def prewarm_running() -> bool:+ worker = _prewarm_worker+ return worker is not None and worker.is_alive()+++def hold_inference() -> Callable[[], None]:+ """``inference_running`` for the event loop: never blocks (the batch in flight, at most ``_PREWARM_BATCH``+ chunks, finishes on its own) and returns the release, to be called exactly once."""+ global _inference_depth+ with _gap:+ _inference_depth += 1+ _prewarm_stop.set()++ def release() -> None:+ global _inference_depth+ with _gap:+ _inference_depth -= 1+ _start_prewarm()++ return release+++def _pause_prewarm() -> None:+ """Stop the gap worker and wait out its batch in flight; it restarts at the next stream's end. Bounded, so a+ wedged embedder can never hold a generation back."""+ _prewarm_stop.set()+ worker = _prewarm_worker+ if worker is not None and worker is not threading.current_thread():+ worker.join(_PAUSE_JOIN_S)+ if worker.is_alive():+ logger.warning("conversation_archive.prewarm_pause_timed_out after=%ss", _PAUSE_JOIN_S)+++def _start_prewarm() -> None:+ global _prewarm_worker+ with _gap:+ if _inference_depth or not _noted_branches:+ return+ if _prewarm_worker is not None and _prewarm_worker.is_alive():+ return+ jobs = list(_noted_branches.values())+ _noted_branches.clear()+ _prewarm_stop.clear()+ _prewarm_worker = threading.Thread(+ target = _prewarm_jobs, args = (jobs,), daemon = True, name = "archive-prewarm"+ )+ _prewarm_worker.start()+++def _other_work_busy() -> bool:+ """Image / video generation or a training run holds the accelerator. Read only from modules already+ imported (never constructs a service), and an unreadable probe counts as busy."""+ import sys++ try:+ for module, attr, probe in (+ ("core.training.training", "_training_backend", "is_training_active"),+ ("core.training.diffusion_training_service", "_service", "is_active"),+ ):+ service = getattr(sys.modules.get(module), attr, None)+ if service is not None and getattr(service, probe)():+ return True+ residency = sys.modules.get("core.inference.gpu_residency")+ if residency is not None:+ for owner in residency.MEDIA_OWNERS:+ if any(residency._media_busy(owner)):+ return True+ except Exception: # noqa: BLE001 -- fail closed: skipping a warm-up costs nothing+ return True+ return False+++def _gap_closed() -> bool:+ return _prewarm_stop.is_set() or _inference_depth > 0 or _other_work_busy()+++def _requeue(jobs: list) -> None:+ """Put back what a closed gap interrupted, behind nothing newer: a fresh note for the same chat wins, and a+ stopped request stays dropped."""+ with _gap:+ kept = {+ (job[0].account_id, job[1]): job+ for job in jobs+ if (job[0].account_id, job[1]) not in _noted_branches and not (job[3] and job[3]())+ }+ merged = {**kept, **_noted_branches}+ _noted_branches.clear()+ _noted_branches.update(list(merged.items())[-_NOTED_MAX:])+++def _prewarm_jobs(jobs: list) -> None:+ from utils.account_context import run_as+ for index, (account, thread_id, messages, cancel_check) in enumerate(jobs):+ if _gap_closed():+ _requeue(jobs[index:])+ return+ # Chat Stop ends this request's background work too, not only its foreground pass.+ stopped = cancel_check or (lambda: False)+ if stopped():+ continue+ try:+ run_as(account, _prewarm_thread, thread_id, messages, stopped)+ except Exception: # noqa: BLE001 -- a cache warm-up is never worth a log storm+ logger.debug(+ "conversation_archive.prewarm_failed thread_id=%s", thread_id, exc_info = True+ )+ if _gap_closed():+ # Possibly cut short; cached vectors make the redo cheap.+ _requeue(jobs[index:])+ return+++def _prewarm_thread(+ thread_id: str,+ messages: list[dict],+ stopped: Callable[[], bool] = lambda: False,+) -> None:+ """Embed this thread's not-yet-archived turns into the vector cache, oldest first (the next to be+ evicted). Writes nothing to the archive, so an edit or rewind cannot strand a stale row."""+ if not can_archive(thread_id):+ return+ from core.inference.context_window import group_turns++ model = config.effective_embedding_model()+ if not _embedder_free():+ return+ identity = embeddings.embedding_identity(model)+ scope = store.conversation_archive_scope(thread_id)+ count = _yielding(embeddings.token_counter(model))+ chunk_limit = embeddings.chunk_tokens(model)+ conn = rag_db.get_connection()+ try:+ budget, todo, done = _PREWARM_MAX_CHUNKS, [], 0+ for group in group_turns(messages):+ if _gap_closed() or stopped() or budget <= 0:+ break+ archivable = _archivable(group)+ text = render_turn(archivable) if archivable else ""+ if not text:+ continue+ digest = hashlib.sha256(text.encode("utf-8", "ignore")).hexdigest()+ if _archived_under(conn, scope, digest, identity):+ continue+ texts = [chunk.text for chunk in _turn_chunks(text, chunk_limit, count)]+ misses = [t for t, v in zip(texts, _cached_vectors(identity, texts)) if v is None]+ todo += misses[:budget]+ budget -= len(misses)+ while len(todo) >= _PREWARM_BATCH and not (_gap_closed() or stopped()):+ done += _prewarm_batch(model, identity, todo[:_PREWARM_BATCH])+ todo = todo[_PREWARM_BATCH:]+ while todo and not (_gap_closed() or stopped()):+ done += _prewarm_batch(model, identity, todo[:_PREWARM_BATCH])+ todo = todo[_PREWARM_BATCH:]+ except _GapClosed:+ pass+ finally:+ conn.close()+ if done:+ logger.info("conversation_archive.prewarmed thread_id=%s chunks=%s", thread_id, done)+++class _GapClosed(Exception):+ pass+++def _yielding(count: Callable[[str], int]) -> Callable[[str], int]:+ def call(text: str) -> int:+ if not _embedder_free():+ raise _GapClosed+ return count(text)++ return call+++def _embedder_busy() -> bool:+ """Whether another caller holds a lock the encode path would wait on: model load / backend build, the ST+ compute lock, or the llama-server backend's serve lock. Probed without blocking."""+ locks = [embeddings._backend_lock, embeddings._lock, embeddings._compute_lock]+ serve = getattr(embeddings._backend, "_serve_lock", None)+ if serve is not None:+ locks.append(serve)+ for lock in locks:+ if not lock.acquire(blocking = False):+ return True+ lock.release()+ return False+++def _embedder_free() -> bool:+ """Wait for the embedder without queueing on its locks: a queued worker cannot see a stop, so a generation+ would wait behind another caller's (e.g. ingestion's) whole batch."""+ while _embedder_busy():+ if _gap_closed():+ return False+ time.sleep(0.05)+ return not _gap_closed()+++def _prewarm_batch(model: str, identity: str, texts: list[str]) -> int:+ if not _embedder_free():+ return 0+ vectors, got = embeddings.encode_with_identity(texts, model_name = model, normalize = True)+ if got != identity:+ # The embedder changed under us; the next request's archive re-embeds in the new space.+ _prewarm_stop.set()+ return 0+ _remember_vectors(got, texts, vectors)+ return len(texts)++def archive_turns(thread_id: str,evicted: list[dict],live: Optional[list[dict]] = None,branch: Optional[list[dict]] = None,+ cancel_check: Optional[Callable[[], bool]] = None,) -> int:"""Index the evicted turns for ``thread_id``. Returns how many were newly written.@@ -373,6 +767,8 @@ def archive_turns(``live`` is the fitted conversation, i.e. what the model can still see. It bounds how manycopies of a repeated turn the archive is allowed to hold right now; see ``_write_budget``.Optional, and omitting it keeps the previous behaviour.++ ``cancel_check`` (Chat Stop) is polled before chunking and between embedding batches."""if not thread_id or not evicted or not enabled():return 0@@ -381,7 +777,19 @@ def archive_turns(if not can_archive(thread_id):logger.debug("conversation_archive.skipped_unpersisted_thread thread_id=%s", thread_id)return 0+ # A request's own compaction goes first and stays marked busy: another slot's stream ending must not+ # restart the gap worker under it. Only a stream's end restarts it, so none starts before this stream.+ with _busy(restart = False):+ return _archive_turns(thread_id, evicted, live, branch, cancel_check)++def _archive_turns(+ thread_id: str,+ evicted: list[dict],+ live: Optional[list[dict]],+ branch: Optional[list[dict]],+ cancel_check: Optional[Callable[[], bool]],+) -> int:# The evictor's own grouper, so an archived unit is exactly the unit the window drops. Imported lazily because the# inference layer imports this module.from core.inference.context_window import group_turns@@ -407,6 +815,8 @@ def archive_turns(written = 0conn = Nonetry:+ if cancel_check is not None and cancel_check():+ return 0count = embeddings.token_counter(model)chunk_limit = Noneconn = rag_db.get_connection()@@ -416,8 +826,12 @@ def archive_turns(# Chunk everything first, then embed it in ONE pass. Per group, a first compaction of a long chat ran dozens# of one-item embedding jobs back to back before the reply could start, and both backends serialise them.- pending = []+ pending, needs, snapshots = [], [], []+ max_chunks = _archive_embed_max_chunks()+ collected = skipped = 0for group, span in groups:+ if cancel_check is not None and cancel_check():+ return 0text = render_turn(group)if not text:continue@@ -442,30 +856,108 @@ def archive_turns()_widen_span(conn, scope, digest, span)continue+ if collected >= max_chunks:+ skipped += 1+ continueif chunk_limit is None:chunk_limit = embeddings.chunk_tokens(model)- chunks = chunk_pages(- [Page(text = text, page_number = None, char_count = len(text))],- max_tokens = chunk_limit,- overlap = config.CHUNK_OVERLAP,- count = count,- )+ chunks = _turn_chunks(text, chunk_limit, count)if chunks:+ # The cap counts what must be encoded now; chunks pre-embedded in an idle gap are free.+ hits = _cached_vectors(expected_identity, [chunk.text for chunk in chunks])+ need = sum(v is None for v in hits)pending.append((group, span, digest, chunks, seats, budget))+ needs.append(need)+ snapshots.append(hits)+ collected += needif not pending:return 0+ if cancel_check is not None and cancel_check():+ logger.info(+ "conversation_archive.embed_cancelled_before_encode thread_id=%s groups=%s",+ thread_id,+ len(pending),+ )+ return 0++ chunk_total = sum(needs)+ if chunk_total > max_chunks or skipped:+ kept, used = [], 0+ for entry, need in zip(pending, needs):+ # On what was kept, not on chunk count: a fully cached first turn adds nothing to `used`.+ if kept and used + need > max_chunks:+ break+ if need > max_chunks and not kept:+ # An oversized first turn still goes, or it would block the backlog forever.+ kept.append(entry)+ used += need+ break+ kept.append(entry)+ used += need+ deferred = len(pending) - len(kept) + skipped+ logger.info(+ "conversation_archive.embed_capped thread_id=%s groups=%s chunks=%s "+ "cap=%s deferred_groups=%s",+ thread_id,+ len(pending),+ chunk_total,+ max_chunks,+ deferred,+ )+ snapshots = snapshots[: len(kept)]+ pending = kept+ if not pending:+ return 0++ texts = [chunk.text for entry in pending for chunk in entry[3]]+ started = time.monotonic()+ logger.info(+ "conversation_archive.embed_start thread_id=%s groups=%s chunks=%s batch=%s",+ thread_id,+ len(pending),+ len(texts),+ _ARCHIVE_EMBED_BATCH,+ )# Identity from the encode that produced these vectors: a concurrent embedder swap would otherwise label them# with a space they were never in.- vectors, identity = embeddings.encode_with_identity(- [chunk.text for entry in pending for chunk in entry[3]],+ vectors, identity, cancelled, embed_error = _embed_pending_chunks(+ texts,model_name = model,- normalize = True,+ cancel_check = cancel_check,+ expected_identity = expected_identity,+ cached = [vector for hits in snapshots for vector in hits],+ max_encode = max(max_chunks, sum(needs[: len(pending)])),)+ logger.info(+ "conversation_archive.embed_done thread_id=%s vectors=%s/%s cancelled=%s elapsed_ms=%s",+ thread_id,+ len(vectors),+ len(texts),+ cancelled,+ int((time.monotonic() - started) * 1000),+ )+ if not vectors:+ return 0++ # Only fully embedded groups; a partial one is left for the next request.+ complete: list = []offset = 0- for group, span, digest, chunks, seats, budget in pending:- group_vectors = vectors[offset : offset + len(chunks)]- offset += len(chunks)+ for entry in pending:+ n = len(entry[3])+ if offset + n > len(vectors):+ break+ complete.append((entry, vectors[offset : offset + n]))+ offset += n+ if len(complete) < len(pending):+ logger.info(+ "conversation_archive.embed_partial thread_id=%s written_groups=%s/%s",+ thread_id,+ len(complete),+ len(pending),+ )++ for (group, span, digest, chunks, seats, budget), group_vectors in complete:roles = " + ".join(dict.fromkeys(str(message.get("role") or "message") for message in group))@@ -591,6 +1083,8 @@ def archive_turns(written += 1if _INGEST_FAILED:globals()["_INGEST_FAILED"] = False+ if embed_error is not None:+ raise embed_errorexcept embeddings.EmbeddingModelDownloadRequiredError as exc:globals()["_INGEST_FAILED"] = True_log_embedder_pending(model, exc)diff --git a/studio/backend/core/training/diffusion_training_service.py b/studio/backend/core/training/diffusion_training_service.pyindex 9db37a7e557..8dbddc39b26 100644--- a/studio/backend/core/training/diffusion_training_service.py+++ b/studio/backend/core/training/diffusion_training_service.py@@ -17,6 +17,7 @@ runs a scripted target on a thread.from __future__ import annotations+from core.inference.inference_gate import pauses_background_embeddingfrom core.training.account_jobs import (account_is_retired,init_job_owner,@@ -665,6 +666,7 @@ class DiffusionTrainingService:@_invalidates_gpu_memory("diffusion training start")@owned_job()+ @pauses_background_embeddingdef start(self, config: dict) -> str:"""Validate ``config``, spawn the trainer, and start pumping its events.diff --git a/studio/backend/core/training/training.py b/studio/backend/core/training/training.pyindex 8e95901766a..ef0898824eb 100644--- a/studio/backend/core/training/training.py+++ b/studio/backend/core/training/training.py@@ -10,6 +10,7 @@ orchestrates the subprocess lifecycle, pumps events from the worker's mp.Queue,exposes the same API to routes/training.py. Pattern follows data_recipe/jobs/manager.py."""+from core.inference.inference_gate import pauses_background_embeddingfrom core.training.account_jobs import (account_is_retired,account_process_spec,@@ -1711,6 +1712,7 @@ class TrainingBackend:@_invalidates_gpu_memory("training start")@owned_job()+ @pauses_background_embeddingdef start_training(self,job_id: str,diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.pyindex 5f6817e7dce..5b482ec4895 100644--- a/studio/backend/routes/inference.py+++ b/studio/backend/routes/inference.py@@ -150,6 +150,7 @@ from core.inference.orchestrator import (_summed_tool_loop_stats,routed_slot,)+from core.inference.inference_gate import gated_async_client, holds_inferencefrom core.inference.llama_admission import (LlamaAdmissionCancelled,LlamaAdmissionConfig,@@ -5986,7 +5987,7 @@ async def _await_stt_disconnect_then_cancel(request, sidecar, cancel_event) -> Ndef _cancelable_nonstreaming_client() -> httpx.AsyncClient:- return httpx.AsyncClient(+ return gated_async_client(limits = httpx.Limits(max_connections = 1, max_keepalive_connections = 0),trust_env = False,verify = _local_ssl_context(),@@ -8335,11 +8336,22 @@ def _direct_llama_request(counted: bool = True):if not counted:yieldreturn+ from core.rag.conversation_archive import _pause_prewarm, hold_inference++ # Inference too (caption / OCR, TTS): background archive embedding stays paused. A worker thread (caption /+ # OCR) also waits out the batch in flight; the TTS route enters this on the event loop, and its generation+ # runs the gated generate_audio_response in a thread, which waits there.+ release = hold_inference()+ try:+ asyncio.get_running_loop()+ except RuntimeError:+ _pause_prewarm()_direct_llama_request_started()try:yieldfinally:_direct_llama_request_finished()+ release()def _direct_llama_is_busy() -> bool:@@ -27461,6 +27473,7 @@ async def _transcribe_audio_bytes()+@holds_inferenceasync def _transcribe_audio_result(raw: bytes,model: Optional[str],@@ -27542,12 +27555,16 @@ async def _transcribe_audio_result()load_options = {"on_phase": on_phase} if on_phase is not None else {}if source_path is not None and timestamps:- await asyncio.to_thread(sidecar.ensure_aligner, model, on_phase, cancel_event)+ await asyncio.to_thread(+ holds_inference(sidecar.ensure_aligner), model, on_phase, cancel_event+ )# Started with its aligner now, or transcribe_path would restart it.load_options["timestamps"] = Trueawait asyncio.to_thread(- functools.partial(- load_stt, model, serving_engine, cancel_event, device = device, **load_options+ holds_inference(+ functools.partial(+ load_stt, model, serving_engine, cancel_event, device = device, **load_options+ )))loaded = getattr(sidecar, "loaded_model", None)@@ -27555,19 +27572,21 @@ async def _transcribe_audio_result(account_access.note_resident_account(f"stt:{serving_engine}", loaded)if source_path is not None:result = await asyncio.to_thread(- functools.partial(- sidecar.transcribe_path,- source_path,- model,- language,- timestamps = timestamps,- cancel_event = cancel_event,- on_phase = on_phase,+ holds_inference(+ functools.partial(+ sidecar.transcribe_path,+ source_path,+ model,+ language,+ timestamps = timestamps,+ cancel_event = cancel_event,+ on_phase = on_phase,+ )))elif on_progress is not None and serving_engine in ("transformers", "mtmd"):result = await asyncio.to_thread(- transcribe,+ holds_inference(transcribe),raw,model,language,@@ -27576,10 +27595,12 @@ async def _transcribe_audio_result(on_progress = on_progress,)elif cancel_event is None:- result = await asyncio.to_thread(transcribe, raw, model, language, fast)+ result = await asyncio.to_thread(+ holds_inference(transcribe), raw, model, language, fast+ )else:result = await asyncio.to_thread(- transcribe,+ holds_inference(transcribe),raw,model,language,@@ -40384,7 +40405,7 @@ async def openai_completions(request: Request, current_subject: str = Depends(gefrom core.inference.llama_cpp import LlamaCppBackendfrom core.inference.llama_keepwarm import mark_response_failed- client = httpx.AsyncClient(+ client = gated_async_client(timeout = _llama_streaming_generation_timeout(),trust_env = False,verify = _local_ssl_context(),@@ -43614,7 +43635,7 @@ async def _responses_stream(_relay_recording_tool_extras(upstream.body_iterator, current_subject))else:- client = httpx.AsyncClient(+ client = gated_async_client(timeout = _llama_streaming_generation_timeout(),trust_env = False,verify = _local_ssl_context(),@@ -48797,7 +48818,7 @@ async def _anthropic_passthrough_stream(# finalizer has anything orphaned to finalize. Each aclose is wrapped in# `try: ... except Exception: pass` so nested anyio cleanup noise can't# bubble out.- client = httpx.AsyncClient(+ client = gated_async_client(timeout = _llama_streaming_generation_timeout(),limits = httpx.Limits(max_keepalive_connections = 0),trust_env = False,@@ -50114,7 +50135,7 @@ async def _openai_passthrough_stream_admitted(# Keep the pre-header window short so accepted SSE clients receive# immediate headers in the common timeout-reduced stall.- client = httpx.AsyncClient(+ client = gated_async_client(timeout = _llama_streaming_generation_timeout(),limits = httpx.Limits(max_keepalive_connections = 0),trust_env = False,diff --git a/studio/backend/routes/managed_engine_chat.py b/studio/backend/routes/managed_engine_chat.pyindex 37d4b5a2cd8..eee446388c9 100644--- a/studio/backend/routes/managed_engine_chat.py+++ b/studio/backend/routes/managed_engine_chat.py@@ -15,6 +15,7 @@ from fastapi import HTTPExceptionfrom fastapi.responses import JSONResponsefrom core.inference.engine_transport import engine_messages, engine_request_timeout+from core.inference.inference_gate import gated_async_clientfrom core.inference.http_stream import closing_response_linesfrom core.inference.studio_tool_loop import ToolLoopPolicy, ToolLoopRun, stream_with_studio_tools@@ -296,7 +297,7 @@ async def managed_tool_chat(tracker.__enter__()async def events():- client = httpx.AsyncClient(trust_env = False, timeout = engine_request_timeout())+ client = gated_async_client(trust_env = False, timeout = engine_request_timeout())watcher = asyncio.create_task(api._await_cancel_or_disconnect_then_close_client(cancel_event = cancel, request = request, client = client@@ -526,7 +527,7 @@ async def managed_engine_completions(request, current_subject, backend, body):cancel = threading.Event()tracker = api._TrackedCancel(cancel, model = public, kind = "completions")tracker.__enter__()- client = httpx.AsyncClient(trust_env = False, timeout = engine_request_timeout())+ client = gated_async_client(trust_env = False, timeout = engine_request_timeout())watcher = asyncio.create_task(api._await_cancel_or_disconnect_then_close_client(cancel_event = cancel, request = request, client = clientdiff --git a/studio/backend/tests/conftest.py b/studio/backend/tests/conftest.pyindex 72d97343c5b..ceb98d4014d 100644--- a/studio/backend/tests/conftest.py+++ b/studio/backend/tests/conftest.py@@ -874,6 +874,23 @@ def dit_train_host(monkeypatch):return dtc[email protected](autouse = True)+def _fresh_archive_vector_cache():+ """The archive's pre-embedded vector cache is process-wide; one test's vectors must not answer another's."""+ archive = sys.modules.get("core.rag.conversation_archive")+ if archive is not None:+ archive._vector_cache.clear()+ archive._noted_branches.clear()+ yield+ archive = sys.modules.get("core.rag.conversation_archive")+ if archive is not None:+ archive._prewarm_stop.set()+ if archive._prewarm_worker is not None:+ archive._prewarm_worker.join()+ archive._vector_cache.clear()+ archive._noted_branches.clear()++@pytest.fixture(autouse = True)def _reset_optional_module_memo():"""_load_optional caches failures too, so one test's fake module would answer the next question."""diff --git a/studio/backend/tests/test_conversation_archive.py b/studio/backend/tests/test_conversation_archive.pyindex 781ec3c60a5..e8b61b67b72 100644--- a/studio/backend/tests/test_conversation_archive.py+++ b/studio/backend/tests/test_conversation_archive.py@@ -1464,6 +1464,883 @@ def test_a_first_compaction_embeds_its_turns_in_one_pass(conn, monkeypatch):assert calls[0] >= 12+def test_archive_embed_honors_cancel_between_batches(conn, monkeypatch):+ """Chat Stop must interrupt the pre-inference SentenceTransformer.encode (#14519)."""+ from core.rag import embeddings++ thread_id = "cancel-embed-thread"+ _save_thread(thread_id, _turn("hello", "hi"), append = True)+ monkeypatch.setattr(conversation_archive, "_ARCHIVE_EMBED_BATCH", 2)+ monkeypatch.setattr(conversation_archive, "_ARCHIVE_EMBED_MAX_CHUNKS_DEFAULT", 10_000)++ calls = []+ real = embeddings.encode_with_identity++ def counted(texts, **kwargs):+ calls.append(len(texts))+ return real(texts, **kwargs)++ monkeypatch.setattr(embeddings, "encode_with_identity", counted)++ evicted = []+ for index in range(8):+ evicted.append({"role": "user", "content": f"cancel question {index} unique {index}"})+ evicted.append({"role": "assistant", "content": f"cancel answer {index} unique {index}"})++ def cancel_check():+ return bool(calls)++ written = conversation_archive.archive_turns(thread_id, evicted, cancel_check = cancel_check)+ assert 0 < written < 8+ assert len(calls) == 1+ assert conversation_archive.archive_turns(thread_id, evicted) == 8 - written+++def test_archive_embed_caps_chunks_per_call(conn, monkeypatch):+ """A huge first compaction must not monopolise the generation path (#14519)."""+ from core.rag import embeddings++ thread_id = "cap-embed-thread"+ _save_thread(thread_id, _turn("hello", "hi"), append = True)+ monkeypatch.setenv("RAG_ARCHIVE_EMBED_MAX_CHUNKS", "4")+ monkeypatch.setattr(conversation_archive, "_ARCHIVE_EMBED_BATCH", 64)++ calls = []+ real = embeddings.encode_with_identity++ def counted(texts, **kwargs):+ calls.append(len(texts))+ return real(texts, **kwargs)++ monkeypatch.setattr(embeddings, "encode_with_identity", counted)++ evicted = []+ for index in range(10):+ evicted.append({"role": "user", "content": f"cap question {index} unique token {index}"})+ evicted.append({"role": "assistant", "content": f"cap answer {index} unique token {index}"})++ first = conversation_archive.archive_turns(thread_id, evicted)+ assert 0 < first < 10+ assert calls and calls[0] <= 4+ total = first+ for _ in range(10):+ written = conversation_archive.archive_turns(thread_id, evicted)+ if not written:+ break+ total += written+ assert total == 10+ assert all(n <= 4 for n in calls)+ assert conversation_archive.archive_turns(thread_id, evicted) == 0+++def test_a_stop_before_chunking_skips_the_embedder_entirely(conn, monkeypatch):+ """Stop already set: no tokenizer load, no chunking, no encode (#14519)."""+ from core.rag import embeddings++ thread_id = "stop-early-thread"+ _save_thread(thread_id, _turn("hello", "hi"), append = True)+ touched = []+ monkeypatch.setattr(embeddings, "token_counter", lambda *_a, **_k: touched.append("count"))+ monkeypatch.setattr(+ conversation_archive, "chunk_pages", lambda *_a, **_k: touched.append("chunk")+ )+ evicted = _turn("early question", "early answer")++ assert conversation_archive.archive_turns(thread_id, evicted, cancel_check = lambda: True) == 0+ assert touched == []+++def test_chunking_stops_once_the_cap_is_reached(conn, monkeypatch):+ """A long backlog must not be tokenized in full when only the cap's worth is embedded."""+ thread_id = "cap-chunk-thread"+ _save_thread(thread_id, _turn("hello", "hi"), append = True)+ monkeypatch.setenv("RAG_ARCHIVE_EMBED_MAX_CHUNKS", "3")+ real = conversation_archive.chunk_pages+ chunked = []++ def counted(*args, **kwargs):+ chunked.append(1)+ return real(*args, **kwargs)++ monkeypatch.setattr(conversation_archive, "chunk_pages", counted)+ evicted = []+ for index in range(20):+ evicted += _turn(f"chunk question {index} unique {index}", f"chunk answer {index}")++ assert conversation_archive.archive_turns(thread_id, evicted) == 3+ assert len(chunked) == 3+++def test_archive_embed_restarts_in_the_new_space_after_a_backend_swap(conn, monkeypatch):+ """Batches before an ST -> llama-server swap must not be written under the old identity."""+ from core.rag import embeddings++ thread_id = "swap-embed-thread"+ _save_thread(thread_id, _turn("hello", "hi"), append = True)+ monkeypatch.setattr(conversation_archive, "_ARCHIVE_EMBED_BATCH", 2)+ monkeypatch.setattr(conversation_archive, "_ARCHIVE_EMBED_MAX_CHUNKS_DEFAULT", 10_000)++ real = embeddings.encode_with_identity+ calls = []++ def swapping(texts, **kwargs):+ calls.append(len(texts))+ vectors, _identity = real(texts, **kwargs)+ return vectors, "sentence-transformers:old" if len(calls) <= 2 else "llama-server:new"++ monkeypatch.setattr(embeddings, "encode_with_identity", swapping)++ evicted = []+ for index in range(8):+ evicted.append({"role": "user", "content": f"swap question {index} unique {index}"})+ evicted.append({"role": "assistant", "content": f"swap answer {index} unique {index}"})++ assert conversation_archive.archive_turns(thread_id, evicted) == 8+ scope = store.conversation_archive_scope(thread_id)+ labels = {+ row[0]+ for row in conn.execute("SELECT embedding_model FROM documents WHERE scope = ?", (scope,))+ }+ assert labels == {"llama-server:new"}+++def _gap_thread(+ thread_id,+ n,+ tag = "gap",+):+ turns = []+ for index in range(n):+ turns += _turn(+ f"{tag} question {index} unique {index}", f"{tag} answer {index} code {index}"+ )+ _save_thread(thread_id, turns, append = True)+ return turns+++def _counting_encoder(monkeypatch):+ from core.rag import embeddings++ calls = []+ real = embeddings.encode_with_identity++ def counted(texts, **kwargs):+ calls.append(list(texts))+ return real(texts, **kwargs)++ monkeypatch.setattr(embeddings, "encode_with_identity", counted)+ return calls+++def _run_gap():+ conversation_archive._start_prewarm()+ worker = conversation_archive._prewarm_worker+ if worker is not None:+ worker.join()+++def test_an_idle_gap_pre_embeds_so_compaction_encodes_nothing(conn, monkeypatch):+ """Turns embedded while llama.cpp is idle cost no encode when they are evicted (#14519)."""+ thread_id = "gap-thread"+ turns = _gap_thread(thread_id, 6)+ calls = _counting_encoder(monkeypatch)++ conversation_archive.note_branch(thread_id, turns)+ _run_gap()+ assert calls+ scope = store.conversation_archive_scope(thread_id)+ # The gap only fills a cache: an edit or rewind before the next request strands nothing.+ assert (+ conn.execute("SELECT COUNT(*) FROM documents WHERE scope = ?", (scope,)).fetchone()[0] == 0+ )++ calls.clear()+ assert conversation_archive.archive_turns(thread_id, turns[:8]) == 4+ assert calls == []+++def test_inference_stops_the_gap_worker_and_it_never_overlaps(conn, monkeypatch):+ """No embedding batch may run while a llama.cpp stream is open."""+ import threading+ import time++ from core.rag import embeddings++ thread_id = "overlap-thread"+ turns = _gap_thread(thread_id, 40, tag = "overlap")+ monkeypatch.setattr(conversation_archive, "_PREWARM_BATCH", 1)+ real = embeddings.encode_with_identity+ started = threading.Event()+ overlapped = []++ def slow(texts, **kwargs):+ started.set()+ overlapped.append(conversation_archive._inference_depth > 0)+ time.sleep(0.02)+ return real(texts, **kwargs)++ monkeypatch.setattr(embeddings, "encode_with_identity", slow)+ conversation_archive.note_branch(thread_id, turns)+ conversation_archive._start_prewarm()+ assert started.wait(5)++ with conversation_archive.inference_running():+ worker = conversation_archive._prewarm_worker+ assert worker is None or not worker.is_alive()+ before = len(overlapped)+ time.sleep(0.1)+ assert len(overlapped) == before+ assert not any(overlapped)+ assert len(overlapped) < 40+++def test_a_foreground_archive_stops_the_gap_worker_first(conn, monkeypatch):+ """A new request's compaction must not queue behind a background job (it runs before the stream opens)."""+ import threading+ import time++ from core.rag import embeddings++ background = "fg-background-thread"+ turns = _gap_thread(background, 40, tag = "fgbg")+ foreground = "fg-foreground-thread"+ fg_turns = _gap_thread(foreground, 2, tag = "fgfg")+ monkeypatch.setattr(conversation_archive, "_PREWARM_BATCH", 1)+ real = embeddings.encode_with_identity+ started = threading.Event()+ worker_calls = []++ def slow(texts, **kwargs):+ if threading.current_thread().name == "archive-prewarm":+ started.set()+ worker_calls.append(len(texts))+ time.sleep(0.02)+ return real(texts, **kwargs)++ monkeypatch.setattr(embeddings, "encode_with_identity", slow)+ conversation_archive.note_branch(background, turns)+ conversation_archive._start_prewarm()+ assert started.wait(5)++ assert conversation_archive.archive_turns(foreground, fg_turns) == 2+ worker = conversation_archive._prewarm_worker+ assert worker is None or not worker.is_alive()+ assert len(worker_calls) < 40+++def test_another_slots_stream_ending_does_not_restart_the_worker_mid_compaction(conn, monkeypatch):+ """With several llama.cpp slots, a stream can end while this request is still compacting."""+ import threading++ from core.rag import embeddings++ thread_id = "slots-thread"+ turns = _gap_thread(thread_id, 2, tag = "slots")+ other = _gap_thread("slots-other", 4, tag = "slotsother")+ real = embeddings.encode_with_identity+ in_archive, release = threading.Event(), threading.Event()++ def held(texts, **kwargs):+ if threading.current_thread().name != "archive-prewarm":+ in_archive.set()+ release.wait(5)+ return real(texts, **kwargs)++ monkeypatch.setattr(embeddings, "encode_with_identity", held)+ fg = threading.Thread(target = conversation_archive.archive_turns, args = (thread_id, turns))+ fg.start()+ assert in_archive.wait(5)+ conversation_archive.note_branch("slots-other", other)+ with conversation_archive.inference_running():+ pass+ worker = conversation_archive._prewarm_worker+ assert worker is None or not worker.is_alive()+ release.set()+ fg.join(5)+++def test_every_inference_entry_point_holds_the_gap_closed(conn, monkeypatch):+ """MLX / Transformers generation, audio and model loads pause the worker too, not only llama streams."""+ import threading++ from core.inference.diffusion import DiffusionBackend+ from core.inference.inference_gate import holds_inference, pauses_background_embedding+ from core.inference.llama_cpp import LlamaCppBackend+ from core.inference.mlx_diffusion_backend import MlxDiffusionBackend+ from core.inference.sd_cpp_backend import SdCppDiffusionBackend+ from core.inference.orchestrator import InferenceOrchestrator+ from core.inference.video import VideoBackend+ from core.training.diffusion_training_service import DiffusionTrainingService+ from core.training.training import TrainingBackend+ from core.inference import stt_registry+ from routes import inference as inference_routes++ for fn in (+ InferenceOrchestrator._generate_dispatched,+ InferenceOrchestrator._generate_inner,+ InferenceOrchestrator.generate_audio_response,+ InferenceOrchestrator.generate_whisper_response,+ InferenceOrchestrator.generate_audio_input_response,+ InferenceOrchestrator.load_model,+ LlamaCppBackend.generate_audio_response,+ LlamaCppBackend.load_model,+ DiffusionBackend.generate,+ SdCppDiffusionBackend.generate,+ MlxDiffusionBackend.generate,+ VideoBackend.generate,+ DiffusionBackend.begin_load,+ SdCppDiffusionBackend.begin_load,+ MlxDiffusionBackend.begin_load,+ VideoBackend.begin_load,+ inference_routes._transcribe_audio_result,+ InferenceOrchestrator.load_stt_model,+ stt_registry.load,+ TrainingBackend.start_training,+ DiffusionTrainingService.start,+ ):+ assert hasattr(fn, "__wrapped__"), fn.__qualname__++ thread_id = "gate-thread"+ turns = _gap_thread(thread_id, 30, tag = "gate")+ monkeypatch.setattr(conversation_archive, "_PREWARM_BATCH", 1)+ calls = _counting_encoder(monkeypatch)+ conversation_archive.note_branch(thread_id, turns)+ conversation_archive._start_prewarm()+ seen = {}++ @holds_inference+ def tokens():+ seen["depth"] = conversation_archive._inference_depth+ worker = conversation_archive._prewarm_worker+ seen["worker_alive"] = worker is not None and worker.is_alive()+ before = len(calls)+ yield 1+ seen["embedded_during"] = len(calls) - before++ @holds_inference+ def load():+ return conversation_archive._inference_depth++ assert list(tokens()) == [1]+ assert seen == {"depth": 1, "worker_alive": False, "embedded_during": 0}+ assert load() == 1+ assert conversation_archive._inference_depth == 0++ # Async entry points (speech to text) hold it too, waiting out the batch off the event loop.+ import asyncio++ conversation_archive._start_prewarm()++ @holds_inference+ async def transcribe():+ worker = conversation_archive._prewarm_worker+ return conversation_archive._inference_depth, worker is not None and worker.is_alive()++ assert asyncio.run(transcribe()) == (1, False)+ assert conversation_archive._inference_depth == 0+ assert threading.current_thread().name != "archive-prewarm"++ # A training start stops the worker before admitting the job; its busy probe keeps it idle after.+ conversation_archive._start_prewarm()++ @pauses_background_embedding+ def start():+ worker = conversation_archive._prewarm_worker+ return worker is not None and worker.is_alive()++ assert start() is False+++def test_an_interrupted_gap_requeues_what_it_did_not_finish(conn, monkeypatch):+ """A closed gap keeps unfinished chats queued for the next one; a stopped request is still dropped."""+ import threading+ import time++ from core.rag import embeddings++ monkeypatch.setattr(conversation_archive, "_PREWARM_BATCH", 1)+ real = embeddings.encode_with_identity+ started = threading.Event()++ def slow(texts, **kwargs):+ started.set()+ time.sleep(0.02)+ return real(texts, **kwargs)++ monkeypatch.setattr(embeddings, "encode_with_identity", slow)+ stop = threading.Event()+ for name in ("requeue-a", "requeue-b", "requeue-stopped"):+ cancel = stop.is_set if name.endswith("stopped") else None+ conversation_archive.note_branch(name, _gap_thread(name, 20, tag = name), cancel)+ conversation_archive._start_prewarm()+ assert started.wait(5)+ stop.set()+ with conversation_archive.inference_running():+ queued = [key[1] for key in conversation_archive._noted_branches]+ # Nothing left for the stream's end to restart, so teardown has no worker to wait out.+ conversation_archive._noted_branches.clear()+ assert queued == ["requeue-a", "requeue-b"]+++def test_a_direct_llama_call_holds_the_gap_closed(conn, monkeypatch):+ """Caption / OCR during ingestion and TTS post to llama-server directly, outside the chat paths."""+ import threading+ import time++ from core.rag import embeddings+ from routes import inference as inference_routes++ # From a worker thread (caption / OCR) the request waits out a batch already in flight.+ real = embeddings.encode_with_identity+ in_batch, encoding = threading.Event(), threading.Event()++ def slow(texts, **kwargs):+ encoding.set()+ try:+ in_batch.set()+ time.sleep(3)+ return real(texts, **kwargs)+ finally:+ encoding.clear()++ monkeypatch.setattr(embeddings, "encode_with_identity", slow)+ monkeypatch.setattr(conversation_archive, "_PREWARM_BATCH", 1)+ conversation_archive.note_branch("direct-thread", _gap_thread("direct-thread", 4, tag = "direct"))+ conversation_archive._start_prewarm()+ assert in_batch.wait(5)+ with inference_routes._direct_llama_request():+ assert not encoding.is_set()+ conversation_archive._noted_branches.clear()++ with inference_routes._direct_llama_request():+ assert conversation_archive._inference_depth == 1+ assert conversation_archive._prewarm_stop.is_set()+ assert conversation_archive._inference_depth == 0+ with inference_routes._direct_llama_request(counted = False):+ assert conversation_archive._inference_depth == 0+++def test_cached_vectors_do_not_keep_their_batch_alive(conn):+ import numpy as np++ batch = np.ones((64, 8), dtype = np.float32)+ conversation_archive._remember_vectors("identity", [f"chunk {i}" for i in range(64)], batch)+ [vector] = conversation_archive._cached_vectors("identity", ["chunk 3"])+ assert vector.base is None and np.array_equal(vector, batch[3])+++def test_a_job_start_keeps_the_gap_closed_until_it_returns(conn, monkeypatch):+ """A stream ending while a load validates (before it publishes its busy state) must not restart the worker."""+ from core.inference.inference_gate import pauses_background_embedding++ thread_id = "start-thread"+ conversation_archive.note_branch(thread_id, _gap_thread(thread_id, 3, tag = "start"))+ calls = _counting_encoder(monkeypatch)++ @pauses_background_embedding+ def begin_load():+ with conversation_archive.inference_running():+ pass+ worker = conversation_archive._prewarm_worker+ return worker is not None and worker.is_alive()++ assert begin_load() is False+ assert calls == []+++def test_a_cancelled_transcription_holds_the_gap_until_its_thread_ends(conn, monkeypatch):+ """Cancelling the request task returns at once; the speech thread it started keeps running."""+ import asyncio+ import threading++ from core.inference.inference_gate import holds_inference++ started, finish = threading.Event(), threading.Event()++ def transcribe():+ started.set()+ finish.wait(5)++ async def route():+ task = asyncio.ensure_future(asyncio.to_thread(holds_inference(transcribe)))+ await asyncio.to_thread(started.wait, 5)+ task.cancel()+ try:+ await task+ except asyncio.CancelledError:+ pass+ held = conversation_archive._inference_depth+ finish.set()+ return held++ # asyncio.run joins the executor on exit, so the thread has ended by then.+ assert asyncio.run(route()) == 1+ assert conversation_archive._inference_depth == 0+++def test_api_passthrough_clients_hold_the_gap_closed_until_closed(conn, monkeypatch):+ """OpenAI / Anthropic / managed-engine passthroughs talk to the engine over their own per-request client."""+ import asyncio+ import gc++ import httpx++ from core.inference import inference_gate++ closed = []++ class FakeClient:+ def __init__(self, **kwargs):+ self.kwargs = kwargs++ async def aclose(self):+ closed.append(self.kwargs)++ # Passthrough tests fake the client class on the httpx module; the gate must build through it.+ monkeypatch.setattr(httpx, "AsyncClient", FakeClient)+ thread_id = "passthrough-thread"+ conversation_archive.note_branch(thread_id, _gap_thread(thread_id, 3, tag = "pass"))+ calls = _counting_encoder(monkeypatch)++ client = inference_gate.gated_async_client(trust_env = False)+ assert isinstance(client, FakeClient)+ assert (+ conversation_archive._inference_depth == 1 and conversation_archive._prewarm_stop.is_set()+ )+ conversation_archive._start_prewarm()+ worker = conversation_archive._prewarm_worker+ assert (worker is None or not worker.is_alive()) and calls == []++ asyncio.run(client.aclose())+ asyncio.run(client.aclose())+ assert closed == [{"trust_env": False}] * 2+ assert conversation_archive._inference_depth == 0++ # The first send waits out a batch already in flight, then goes out.+ import threading+ import time++ from core.rag import embeddings++ real = embeddings.encode_with_identity+ in_batch, encoding, sent_mid_batch = threading.Event(), threading.Event(), []++ def slow(texts, **kwargs):+ encoding.set()+ try:+ in_batch.set()+ # Long enough to still be running when the request goes out on a loaded host.+ time.sleep(3)+ return real(texts, **kwargs)+ finally:+ encoding.clear()++ class SendingClient(FakeClient):+ async def send(self, request):+ sent_mid_batch.append(encoding.is_set())+ return "response"++ monkeypatch.setattr(httpx, "AsyncClient", SendingClient)+ monkeypatch.setattr(embeddings, "encode_with_identity", slow)+ monkeypatch.setattr(conversation_archive, "_PREWARM_BATCH", 1)+ conversation_archive.note_branch(thread_id, _gap_thread(thread_id, 6, tag = "send"))+ conversation_archive._start_prewarm()+ assert in_batch.wait(5)+ client = inference_gate.gated_async_client()+ assert asyncio.run(client.send("request")) == "response"+ assert sent_mid_batch == [False]+ conversation_archive._noted_branches.clear()+ asyncio.run(client.aclose())+ monkeypatch.setattr(httpx, "AsyncClient", FakeClient)++ # A path that never closes its client still reopens the gap once it is collected.+ inference_gate.gated_async_client()+ gc.collect()+ assert conversation_archive._inference_depth == 0++[email protected]("held", ["compute", "llama_serve", "backend_build"])+def test_the_gap_worker_never_queues_behind_another_embedder_user(conn, monkeypatch, held):+ """While ingestion holds the embedder, a new generation must not wait for the worker to get and finish its turn."""+ import time++ from core.rag import embeddings++ thread_id = "lock-thread"+ turns = _gap_thread(thread_id, 6, tag = "lock")+ real = embeddings.encode_with_identity++ import threading+ import types++ # The llama-server embedder serialises on its own reentrant lock, not the ST one.+ serve = threading.RLock()+ monkeypatch.setattr(embeddings, "_backend", types.SimpleNamespace(_serve_lock = serve))+ lock = {+ "compute": embeddings._compute_lock,+ "llama_serve": serve,+ "backend_build": embeddings._backend_lock,+ }[held]++ def locked(texts, **kwargs):+ with lock:+ time.sleep(0.5)+ return real(texts, **kwargs)++ monkeypatch.setattr(embeddings, "encode_with_identity", locked)+ monkeypatch.setattr(conversation_archive, "_PAUSE_JOIN_S", 30.0)+ conversation_archive.note_branch(thread_id, turns)+ with lock:+ conversation_archive._start_prewarm()+ time.sleep(0.3)+ started = time.monotonic()+ with conversation_archive.inference_running():+ waited = time.monotonic() - started+ assert waited < 1.0++[email protected]("busy", ["training", "diffusion_training", "media", "unreadable"])+def test_image_video_or_training_work_keeps_the_gap_closed(conn, monkeypatch, busy):+ """Non-LLM accelerator work pauses the worker too; an unreadable probe counts as busy."""+ import sys+ import types++ class Service:+ def is_training_active(self):+ return busy == "training"++ def is_active(self):+ return busy == "diffusion_training"++ training = types.SimpleNamespace(_training_backend = Service())+ diffusion = types.SimpleNamespace(_service = Service())++ def media_busy(owner):+ if busy == "unreadable":+ raise RuntimeError("probe failed")+ return (False, busy == "media")++ residency = types.SimpleNamespace(MEDIA_OWNERS = ("diffusion", "video"), _media_busy = media_busy)+ monkeypatch.setitem(sys.modules, "core.training.training", training)+ monkeypatch.setitem(sys.modules, "core.training.diffusion_training_service", diffusion)+ monkeypatch.setitem(sys.modules, "core.inference.gpu_residency", residency)++ thread_id = f"busy-{busy}"+ turns = _gap_thread(thread_id, 3, tag = busy)+ calls = _counting_encoder(monkeypatch)+ conversation_archive.note_branch(thread_id, turns)+ _run_gap()+ assert calls == []+++def test_the_gap_opens_when_nothing_else_is_running(conn, monkeypatch):+ import sys+ import types++ idle = types.SimpleNamespace(is_training_active = lambda: False, is_active = lambda: False)+ monkeypatch.setitem(+ sys.modules, "core.training.training", types.SimpleNamespace(_training_backend = idle)+ )+ monkeypatch.setitem(+ sys.modules,+ "core.training.diffusion_training_service",+ types.SimpleNamespace(_service = idle),+ )+ monkeypatch.setitem(+ sys.modules,+ "core.inference.gpu_residency",+ types.SimpleNamespace(+ MEDIA_OWNERS = ("diffusion",), _media_busy = lambda owner: (False, False)+ ),+ )+ thread_id = "idle-thread"+ turns = _gap_thread(thread_id, 3, tag = "idle")+ calls = _counting_encoder(monkeypatch)+ conversation_archive.note_branch(thread_id, turns)+ _run_gap()+ assert calls+++def test_an_edited_turn_misses_the_cache_and_is_embedded_fresh(conn, monkeypatch):+ thread_id = "edit-thread"+ turns = _gap_thread(thread_id, 2, tag = "edit")+ calls = _counting_encoder(monkeypatch)+ conversation_archive.note_branch(thread_id, turns)+ _run_gap()++ edited = [dict(m) for m in turns]+ edited[1]["content"] = "edit answer 0 was revised to code 77"+ _save_thread(thread_id, edited, append = False)+ calls.clear()+ assert conversation_archive.archive_turns(thread_id, edited[:2]) == 1+ assert [text for batch in calls for text in batch] and all(+ "code 77" in text for batch in calls for text in batch+ )+++def test_the_cap_counts_only_chunks_that_still_need_embedding(conn, monkeypatch):+ thread_id = "cap-cache-thread"+ turns = _gap_thread(thread_id, 6, tag = "capcache")+ monkeypatch.setenv("RAG_ARCHIVE_EMBED_MAX_CHUNKS", "2")+ conversation_archive.note_branch(thread_id, turns)+ _run_gap()++ calls = _counting_encoder(monkeypatch)+ assert conversation_archive.archive_turns(thread_id, turns) == 6+ assert calls == []+++def test_an_oversized_turn_after_a_cached_one_waits_for_the_next_compaction(conn, monkeypatch):+ """A pre-embedded first turn counts nothing toward the cap; the huge turn behind it must still be deferred."""+ thread_id = "cap-cached-then-huge"+ cached = _gap_thread(thread_id, 1, tag = "cachedfirst")+ huge = _turn("paste this log", " ".join(f"line{i}" for i in range(6000)))+ _save_thread(thread_id, huge, append = True)+ monkeypatch.setenv("RAG_ARCHIVE_EMBED_MAX_CHUNKS", "2")+ conversation_archive.note_branch(thread_id, cached)+ _run_gap()++ calls = _counting_encoder(monkeypatch)+ assert conversation_archive.archive_turns(thread_id, cached + huge) == 1+ assert calls == []+++def test_a_gap_job_runs_as_the_account_that_noted_it(monkeypatch):+ """The worker is a plain thread; without the binding it would read the owner's databases."""+ from utils.account_context import AccountContext, current_account, run_as++ seen = []+ monkeypatch.setattr(conversation_archive, "enabled", lambda: True)+ monkeypatch.setattr(+ conversation_archive,+ "_prewarm_thread",+ lambda thread_id, messages, stopped: seen.append((current_account().account_id, thread_id)),+ )+ member = AccountContext("acct-member", "member", "user")+ run_as(+ member,+ conversation_archive.note_branch,+ "member-thread",+ [{"role": "user", "content": "x"}],+ )+ _run_gap()+ assert seen == [("acct-member", "member-thread")]+++def test_a_stopped_request_leaves_no_background_work(conn, monkeypatch):+ """Stop before the gap: the noted job is dropped, not resumed in the background."""+ thread_id = "stopped-gap-thread"+ turns = _gap_thread(thread_id, 4, tag = "stopped")+ calls = _counting_encoder(monkeypatch)+ conversation_archive.note_branch(thread_id, turns, lambda: True)+ _run_gap()+ assert calls == []+++def test_stop_during_the_gap_ends_the_background_work(conn, monkeypatch):+ thread_id = "stop-mid-gap-thread"+ turns = _gap_thread(thread_id, 20, tag = "midgap")+ monkeypatch.setattr(conversation_archive, "_PREWARM_BATCH", 1)+ calls = _counting_encoder(monkeypatch)+ conversation_archive.note_branch(thread_id, turns, lambda: len(calls) >= 2)+ _run_gap()+ assert len(calls) == 2+++def test_the_cap_holds_when_the_cache_churns_before_the_encode(conn, monkeypatch):+ """Vectors counted as free stay free even if the cache evicts them before encoding."""+ thread_id = "churn-thread"+ turns = _gap_thread(thread_id, 6, tag = "churn")+ monkeypatch.setenv("RAG_ARCHIVE_EMBED_MAX_CHUNKS", "2")+ conversation_archive.note_branch(thread_id, turns)+ _run_gap()++ real = conversation_archive._embed_pending_chunks++ def evict_then_embed(*args, **kwargs):+ conversation_archive._vector_cache.clear()+ return real(*args, **kwargs)++ monkeypatch.setattr(conversation_archive, "_embed_pending_chunks", evict_then_embed)+ calls = _counting_encoder(monkeypatch)+ assert conversation_archive.archive_turns(thread_id, turns) == 6+ assert calls == []+++def test_an_embedder_swap_restart_still_honours_the_cap(conn, monkeypatch):+ """A restart in the new space re-encodes former cache hits; it must stay within the request's cap."""+ from core.rag import embeddings++ thread_id = "swap-cap-thread"+ turns = _gap_thread(thread_id, 6, tag = "swapcap")+ conversation_archive.note_branch(thread_id, turns)+ _run_gap()+ extra = _turn("swapcap fresh question", "swapcap fresh answer")+ _save_thread(thread_id, extra, append = True)+ monkeypatch.setenv("RAG_ARCHIVE_EMBED_MAX_CHUNKS", "2")++ real = embeddings.encode_with_identity+ calls = []++ def swapped(texts, **kwargs):+ calls.append(len(texts))+ vectors, _identity = real(texts, **kwargs)+ return vectors, "llama-server:swapped"++ monkeypatch.setattr(embeddings, "encode_with_identity", swapped)+ conversation_archive.archive_turns(thread_id, turns + extra)+ assert calls[0] == 1+ assert sum(calls[1:]) <= 2+++def test_noted_snapshots_of_stopped_requests_do_not_pile_up(monkeypatch):+ monkeypatch.setattr(conversation_archive, "enabled", lambda: True)+ message = [{"role": "user", "content": "x"}]+ conversation_archive.note_branch("stopped-a", message, lambda: True)+ conversation_archive.note_branch("live-b", message)+ assert [key[1] for key in conversation_archive._noted_branches] == ["live-b"]+ for index in range(20):+ conversation_archive.note_branch(f"many-{index}", message)+ assert len(conversation_archive._noted_branches) <= conversation_archive._NOTED_MAX+ # Re-noting a queued thread replaces its snapshot without evicting an unrelated chat.+ queued = list(conversation_archive._noted_branches)+ conversation_archive.note_branch(queued[-1][1], message)+ assert set(conversation_archive._noted_branches) == set(queued)+++def test_archive_embed_failure_keeps_completed_batches(conn, monkeypatch):+ """A batch failing late in a long pass must not discard the batches before it (#14519)."""+ from core.rag import embeddings++ thread_id = "fail-embed-thread"+ _save_thread(thread_id, _turn("hello", "hi"), append = True)+ monkeypatch.setattr(conversation_archive, "_ARCHIVE_EMBED_BATCH", 2)+ monkeypatch.setattr(conversation_archive, "_ARCHIVE_EMBED_MAX_CHUNKS_DEFAULT", 10_000)++ real = embeddings.encode_with_identity+ calls = []++ def flaky(texts, **kwargs):+ calls.append(len(texts))+ if len(calls) == 3:+ raise TimeoutError("embedder timed out")+ return real(texts, **kwargs)++ monkeypatch.setattr(embeddings, "encode_with_identity", flaky)++ evicted = []+ for index in range(8):+ evicted.append({"role": "user", "content": f"fail question {index} unique {index}"})+ evicted.append({"role": "assistant", "content": f"fail answer {index} unique {index}"})++ written = conversation_archive.archive_turns(thread_id, evicted)+ assert 0 < written < 8+ assert conversation_archive.degraded()+ calls.clear()+ assert conversation_archive.archive_turns(thread_id, evicted) == 8 - written+ assert sum(calls) <= 8 - written++VARIABLE = "ZQXVARA123"diff --git a/studio/backend/tests/test_conversation_recall_injection.py b/studio/backend/tests/test_conversation_recall_injection.pyindex 898ae7dfaf7..9d0d78cac97 100644--- a/studio/backend/tests/test_conversation_recall_injection.py+++ b/studio/backend/tests/test_conversation_recall_injection.py@@ -165,6 +165,28 @@ def test_archive_and_recall_skips_recall_once_already_done(archived):assert "recalled_chunks" not in result["counts"][email protected]("stopped", [False, True])+def test_a_stopped_request_skips_the_automatic_recall(archived, monkeypatch, stopped):+ """Recall embeds the query; a Stopped request must not pay for it (#14519)."""+ recalls = []+ monkeypatch.setattr(tools_mod, "build_conversation_recall", lambda *_a, **_k: recalls.append(1))+ before = _conversation() + [{"role": "user", "content": "evicted turn"}]+ after = _conversation()++ result = llama_cpp._archive_and_recall(+ after,+ before,+ thread_id = THREAD,+ style = "tool",+ recall_done = False,+ recall_budget_tokens = 100_000,+ cancel_check = lambda: stopped,+ )++ assert recalls == ([] if stopped else [1])+ assert result["recalled"] is False++def test_a_refused_fit_still_archives_what_it_evicted(archived):"""`recall_done` skips recall without skipping the archive write."""before = _conversation() + [{"role": "user", "content": "a turn only the fit saw"}]diff --git a/studio/backend/tests/test_mlx_context_compaction.py b/studio/backend/tests/test_mlx_context_compaction.pyindex 6403a8cf451..acdcf38f3ac 100644--- a/studio/backend/tests/test_mlx_context_compaction.py+++ b/studio/backend/tests/test_mlx_context_compaction.py@@ -87,6 +87,19 @@ def test_an_overflowing_prompt_drops_its_oldest_turns_and_reports_the_boundary(assert result["messages"][-1] is messages[-1]+def test_chat_stop_reaches_the_archive_embedding_pass(archive_calls):+ """Stop must be able to interrupt the archive's embedding before inference (#14519)."""+ import threading++ stop = threading.Event()+ _fit(_backend(), _thread(), thread_id = "thread-1", cancel_event = stop)++ cancel_check = archive_calls[-1]["cancel_check"]+ assert cancel_check() is False+ stop.set()+ assert cancel_check() is True++@pytest.mark.parametrize("case", ["not_opted_in", "not_mlx", "media", "resumed_reply", "resumed_thought", "no_count"])