Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: cancel + bound conversation-archive embeddings before inference (#14519) (#14546)

Indrajit swain7dbc8b9ec890
diff --git a/studio/backend/core/inference/diffusion.py b/studio/backend/core/inference/diffusion.py
index f05a623562a..27bdfc9996b 100644
--- a/studio/backend/core/inference/diffusion.py
+++ b/studio/backend/core/inference/diffusion.py
@@ -35,6 +35,7 @@ from dataclasses import dataclass, field, replace
from pathlib import Path
from typing import Any, Callable, Mapping, Optional, Sequence
+from core.inference.inference_gate import holds_inference, pauses_background_embedding
from core._torchao_stub import (
alias_torchao_fqn_to_config,
hide_xformers_built_for_another_torch,
@@ -4722,6 +4723,7 @@ class DiffusionBackend:
assert_pick_is_not_speech(repo_id, gguf_filename, hf_token, allow_network)
@_account_owned_load
+ @pauses_background_embedding
def begin_load(
self,
repo_id: str,
@@ -12582,6 +12584,7 @@ class DiffusionBackend:
)
@_release_render_on_unload
+ @holds_inference
@with_hires
def generate(
self,
diff --git a/studio/backend/core/inference/inference_gate.py b/studio/backend/core/inference/inference_gate.py
new file mode 100644
index 00000000000..cc5fdff0c64
--- /dev/null
+++ b/studio/backend/core/inference/inference_gate.py
@@ -0,0 +1,100 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Mark a function as inference so background archive embedding never runs alongside it."""
+
+from __future__ import annotations
+
+import functools
+import inspect
+import weakref
+
+import httpx
+
+
+def holds_inference(fn):
+ """The archive's gap worker pauses (waiting out its batch in flight) for ``fn``'s whole run; a generator
+ holds it until exhausted or closed. Imported lazily: this module must stay free of the RAG stack."""
+ if inspect.iscoroutinefunction(fn):
+
+ @functools.wraps(fn)
+ async def coroutine(*args, **kwargs):
+ import asyncio
+
+ from core.rag.conversation_archive import _pause_prewarm, hold_inference
+
+ release = hold_inference()
+ try:
+ # The join waits out the batch in flight, so off the event loop.
+ await asyncio.to_thread(_pause_prewarm)
+ return await fn(*args, **kwargs)
+ finally:
+ release()
+
+ return coroutine
+
+ if inspect.isgeneratorfunction(fn):
+
+ @functools.wraps(fn)
+ def generator(*args, **kwargs):
+ from core.rag.conversation_archive import inference_running
+ with inference_running():
+ yield from fn(*args, **kwargs)
+
+ return generator
+
+ @functools.wraps(fn)
+ def call(*args, **kwargs):
+ from core.rag.conversation_archive import inference_running
+ with inference_running():
+ return fn(*args, **kwargs)
+
+ return call
+
+
+def pauses_background_embedding(fn):
+ """For starts of long jobs (training): stop the gap worker and wait out its batch before admitting the job;
+ the worker's own busy probe keeps it idle for the job's lifetime."""
+
+ @functools.wraps(fn)
+ def call(*args, **kwargs):
+ from core.rag.conversation_archive import _busy
+
+ # Held across the start, so a stream ending meanwhile cannot restart the worker before the job publishes
+ # the state the busy probe reads.
+ with _busy(restart = False):
+ return fn(*args, **kwargs)
+
+ return call
+
+
+def gated_async_client(*args, **kwargs):
+ """A per-request ``httpx.AsyncClient`` to a local engine (API passthroughs): background embedding stays paused
+ until it is ``aclose``d, or collected if a path never closes it. Built through ``httpx.AsyncClient`` at call
+ time, so a patched client class is honoured."""
+ from core.rag.conversation_archive import hold_inference
+
+ client = httpx.AsyncClient(*args, **kwargs)
+ release = weakref.finalize(client, hold_inference())
+ close = client.aclose
+ send = getattr(client, "send", None)
+
+ async def gated_send(*send_args, **send_kwargs):
+ # The first request waits out the batch in flight (off the event loop); httpx routes every verb and
+ # stream() through send.
+ from core.rag.conversation_archive import _pause_prewarm, prewarm_running
+ if prewarm_running():
+ import asyncio
+ await asyncio.to_thread(_pause_prewarm)
+ return await send(*send_args, **send_kwargs)
+
+ async def aclose():
+ try:
+ return await close()
+ finally:
+ release()
+
+ client.aclose = aclose
+ if inspect.iscoroutinefunction(send):
+ client.send = gated_send
+ return client
diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.py
index d5f9942330f..fd78923ff4f 100644
--- a/studio/backend/core/inference/llama_cpp.py
+++ b/studio/backend/core/inference/llama_cpp.py
@@ -22,6 +22,7 @@ import math
import os
import re
import struct
+from core.inference.inference_gate import holds_inference
from loggers import get_logger
from utils.gpu_memory_events import invalidate_gpu_memory as _invalidate_gpu_memory
from utils import host_memory as _host_memory
@@ -2289,6 +2290,8 @@ def _archive_and_recall(
recall_budget_tokens: int = 0,
count_tokens: Optional[Callable[[list[dict]], int]] = None,
branch_messages: Optional[list[dict]] = None,
+ cancel_check: Optional[Callable[[], bool]] = None,
+ background: bool = False,
) -> dict:
"""Keep the turns this fit evicted, then pull the relevant ones back.
@@ -2313,19 +2316,26 @@ def _archive_and_recall(
return result
gone = evicted_messages(before, conversation)
+ if background:
+ # llama.cpp only: its streams gate the idle-gap worker (`inference_running`).
+ conversation_archive.note_branch(thread_id, before, cancel_check)
# `conversation` is the fitted prompt, so it says which repeats of a turn the model
# can still read. Without it a twice-said turn is archived twice and spends two
# recall slots on text already in the prompt.
archived = (
conversation_archive.archive_turns(
- thread_id, gone, live = conversation, branch = branch_messages or conversation
+ thread_id,
+ gone,
+ live = conversation,
+ branch = branch_messages or conversation,
+ cancel_check = cancel_check,
)
if gone
else 0
)
counts = {"archived_messages": archived}
- if recall_done:
+ if recall_done or (cancel_check is not None and cancel_check()):
result["counts"] = counts
return result
if not force_recall:
@@ -25938,6 +25948,7 @@ class LlamaCppBackend:
@_invalidates_gpu_memory("llama.cpp load")
@_with_gguf_load_marker
+ @holds_inference
def load_model(
self,
intent: GgufLoadIntent,
@@ -39325,13 +39336,18 @@ class LlamaCppBackend:
or getattr(self, "_chat_template", None),
),
}
+ from core.rag.conversation_archive import inference_running
+
stream_timeout = httpx.Timeout(connect = 10, read = 0.5, write = 10, pool = 10)
- with httpx.Client(
- timeout = stream_timeout,
- limits = httpx.Limits(max_keepalive_connections = 0),
- trust_env = False,
- verify = _local_ssl_context(),
- ) as client:
+ with (
+ inference_running(),
+ httpx.Client(
+ timeout = stream_timeout,
+ limits = httpx.Limits(max_keepalive_connections = 0),
+ trust_env = False,
+ verify = _local_ssl_context(),
+ ) as client,
+ ):
first_token_deadline = start_first_token_wait()
with self._stream_with_retry(
client,
@@ -40065,6 +40081,8 @@ class LlamaCppBackend:
chat_template_kwargs = _reasoning_kw,
continue_final_message = continue_final_message,
),
+ cancel_check = lambda: bool(cancel_event and cancel_event.is_set()),
+ background = True,
)
openai_messages = _recalled["conversation"]
truncation = {**truncation, **_recalled["counts"]}
@@ -41139,6 +41157,8 @@ class LlamaCppBackend:
strict = True,
chat_template_kwargs = _reasoning_kw,
),
+ cancel_check = lambda: bool(cancel_event and cancel_event.is_set()),
+ background = True,
)
conversation = _recalled["conversation"]
truncation = {**truncation, **_recalled["counts"]}
@@ -41300,6 +41320,8 @@ class LlamaCppBackend:
style = "inline",
recall_done = True,
branch_messages = _extend_live_branch(conversation),
+ cancel_check = lambda: bool(cancel_event and cancel_event.is_set()),
+ background = True,
)["counts"]
)
payload["messages"] = neutralize_control_markup_in_messages(
@@ -43990,6 +44012,8 @@ class LlamaCppBackend:
strict = True,
chat_template_kwargs = _reasoning_kw,
),
+ cancel_check = lambda: bool(cancel_event and cancel_event.is_set()),
+ background = True,
)
conversation = _recalled["conversation"]
truncation = {**truncation, **_recalled["counts"]}
@@ -44134,6 +44158,8 @@ class LlamaCppBackend:
style = "inline",
recall_done = True,
branch_messages = _extend_live_branch(conversation),
+ cancel_check = lambda: bool(cancel_event and cancel_event.is_set()),
+ background = True,
)["counts"]
)
# Recorded rather than only forwarded, for the same reason as the
@@ -45120,6 +45146,7 @@ class LlamaCppBackend:
LlamaCppBackend._codec_mgr.load_codec(audio_type, device, model_repo_path = model_repo_path)
logger.info(f"Loaded audio codec for GGUF TTS: {audio_type}")
+ @holds_inference
def generate_audio_response(
self,
text: str,
diff --git a/studio/backend/core/inference/mlx_diffusion_backend.py b/studio/backend/core/inference/mlx_diffusion_backend.py
index 9a6f3fb9b7b..d398c061998 100644
--- a/studio/backend/core/inference/mlx_diffusion_backend.py
+++ b/studio/backend/core/inference/mlx_diffusion_backend.py
@@ -20,6 +20,7 @@ from dataclasses import dataclass, field
from pathlib import Path
from typing import Any, Callable, Optional, Union
+from core.inference.inference_gate import holds_inference, pauses_background_embedding
from core.inference.diffusion_families import (
DIFFUSION_CANCELLED_MSG,
DIFFUSION_NOT_LOADED_MSG,
@@ -880,6 +881,7 @@ class MlxDiffusionBackend:
# ── Load ─────────────────────────────────────────────────────────────────
+ @pauses_background_embedding
def begin_load(
self,
repo_id: str,
@@ -1235,6 +1237,7 @@ class MlxDiffusionBackend:
# ── Generate ─────────────────────────────────────────────────────────────
+ @holds_inference
def generate(
self,
*,
diff --git a/studio/backend/core/inference/orchestrator.py b/studio/backend/core/inference/orchestrator.py
index ea8f2bae513..92fe0c0edab 100644
--- a/studio/backend/core/inference/orchestrator.py
+++ b/studio/backend/core/inference/orchestrator.py
@@ -14,6 +14,7 @@ import contextlib
import os
import signal
from loggers import get_logger
+from core.inference.inference_gate import holds_inference
from utils.gpu_memory_events import invalidates_gpu_memory as _invalidates_gpu_memory
import multiprocessing as mp
import queue
@@ -1743,6 +1744,7 @@ class InferenceOrchestrator:
logger.exception("Inference dispatcher: failed to route a response; continuing")
continue
+ @holds_inference
def _generate_dispatched(
self,
messages: list = None,
@@ -2068,6 +2070,7 @@ class InferenceOrchestrator:
load_generation: int = 0
@_invalidates_gpu_memory("inference load")
+ @holds_inference
def load_model(
self,
config,
@@ -2675,6 +2678,7 @@ class InferenceOrchestrator:
# Dictation models run in the STT sidecars (whisper-server, llama-server, and the Transformers spawn child), not
# the chat worker. Their lifecycle goes through here all the same, so one object knows everything that is resident
# and Voice settings and Model Hub cannot report different things about one model.
+ @holds_inference
def load_stt_model(
self,
model: Optional[str],
@@ -3041,6 +3045,7 @@ class InferenceOrchestrator:
reply_returns = calls_tools,
),
count_tokens = _count,
+ cancel_check = lambda: bool(cancel_event is not None and cancel_event.is_set()),
)
fitted = recall["conversation"]
truncation = {**truncation, **recall["counts"]}
@@ -3592,6 +3597,7 @@ class InferenceOrchestrator:
if callable(close):
close()
+ @holds_inference
def _generate_inner(
self,
messages: list = None,
@@ -3894,6 +3900,7 @@ class InferenceOrchestrator:
)
return outputs
+ @holds_inference
def generate_audio_response(
self,
text: str,
@@ -4145,6 +4152,7 @@ class InferenceOrchestrator:
self._release_worker(cancel_event)
release_mailbox()
+ @holds_inference
def generate_whisper_response(
self,
audio_array,
@@ -4165,6 +4173,7 @@ class InferenceOrchestrator:
extra_audio_arrays = extra_audio_arrays,
)
+ @holds_inference
def generate_audio_input_response(
self,
messages,
diff --git a/studio/backend/core/inference/sd_cpp_backend.py b/studio/backend/core/inference/sd_cpp_backend.py
index f7a56ef94d1..742c3611ad6 100644
--- a/studio/backend/core/inference/sd_cpp_backend.py
+++ b/studio/backend/core/inference/sd_cpp_backend.py
@@ -34,6 +34,7 @@ from dataclasses import dataclass, field, replace
from pathlib import Path
from typing import Any, Optional
+from core.inference.inference_gate import holds_inference, pauses_background_embedding
from core.inference.diffusion_auto_policy import build_resolved_record, format_generation_for_log
from core.inference.diffusion_compat import flux2_inner_dim_for_pick
from core.inference.diffusion_device import (
@@ -3257,6 +3258,7 @@ class SdCppDiffusionBackend:
)
return upgraded
+ @pauses_background_embedding
def begin_load(
self,
repo_id: str,
@@ -4423,6 +4425,7 @@ class SdCppDiffusionBackend:
and workflow in (None, "reference")
)
+ @holds_inference
def generate(
self,
*,
diff --git a/studio/backend/core/inference/stt_registry.py b/studio/backend/core/inference/stt_registry.py
index 37932dedd8f..3f43312b097 100644
--- a/studio/backend/core/inference/stt_registry.py
+++ b/studio/backend/core/inference/stt_registry.py
@@ -15,6 +15,7 @@ from __future__ import annotations
import threading
from typing import Any, Optional, Sequence
+from core.inference.inference_gate import holds_inference
from loggers import get_logger
from utils.gpu_memory_events import invalidates_gpu_memory as _invalidates_gpu_memory
@@ -45,6 +46,7 @@ def sidecar_for(engine: str) -> Any:
@_invalidates_gpu_memory("stt load")
+@holds_inference
def load(
model: Optional[str],
engine: str,
diff --git a/studio/backend/core/inference/video.py b/studio/backend/core/inference/video.py
index 81302a7e8c5..9754b54a24c 100644
--- a/studio/backend/core/inference/video.py
+++ b/studio/backend/core/inference/video.py
@@ -296,6 +296,8 @@ from .video_nvenc import encode_nvenc, nvenc_gpu
from .video_stream_encode import claim_stream, offer_stream, open_stream_encode
from utils.hardware import clear_gpu_cache
+from core.inference.inference_gate import holds_inference, pauses_background_embedding
+
# Shared with the image backend so both pin every loader call to the same live cache root
from core.inference.diffusion import hub_cache_dir
@@ -4384,6 +4386,7 @@ class VideoBackend:
# ── background load + progress ───────────────────────────────────────────
+ @pauses_background_embedding
def begin_load(
self,
repo_id: str,
@@ -11598,6 +11601,7 @@ class VideoBackend:
video_id = (video or {}).get("id"),
)
+ @holds_inference
def generate(
self,
*,
diff --git a/studio/backend/core/rag/conversation_archive.py b/studio/backend/core/rag/conversation_archive.py
index 6ee7ee4dfd7..caceee0b3c3 100644
--- a/studio/backend/core/rag/conversation_archive.py
+++ b/studio/backend/core/rag/conversation_archive.py
@@ -20,8 +20,13 @@ import hashlib
import logging
import bisect
import json
+import os
import re
-from typing import Optional
+import threading
+import time
+from collections import OrderedDict
+from contextlib import contextmanager
+from typing import Callable, Optional
from storage import rag_db
@@ -31,6 +36,17 @@ from .parsers import Page
logger = logging.getLogger(__name__)
+_ARCHIVE_EMBED_BATCH = 64
+# Per call; the rest is archived by later requests, which re-evict the same turns.
+_ARCHIVE_EMBED_MAX_CHUNKS_DEFAULT = 256
+# Background pre-embedding runs only while no inference runs (tool calls, between turns); small batches so
+# a new generation waits at most one of them.
+_PREWARM_BATCH = 16
+_PREWARM_MAX_CHUNKS = 2048
+_VECTOR_CACHE_MAX = 8192
+_NOTED_MAX = 8
+_PAUSE_JOIN_S = 10.0
+
# Instructions, not conversation. An archived system prompt could come back quoted as an "earlier turn" and read as
# user-authored text.
_SKIP_ROLES = frozenset({"system", "developer"})
@@ -358,11 +374,389 @@ def can_archive(thread_id: Optional[str]) -> bool:
return False
+def _archive_embed_max_chunks() -> int:
+ raw = os.environ.get("RAG_ARCHIVE_EMBED_MAX_CHUNKS", "").strip()
+ if not raw:
+ return _ARCHIVE_EMBED_MAX_CHUNKS_DEFAULT
+ try:
+ return max(1, int(raw))
+ except ValueError:
+ return _ARCHIVE_EMBED_MAX_CHUNKS_DEFAULT
+
+
+def _chunk_key(identity: str, text: str) -> tuple[str, str]:
+ return identity, hashlib.sha256(text.encode("utf-8", "ignore")).hexdigest()
+
+
+_vector_cache: "OrderedDict[tuple[str, str], object]" = OrderedDict()
+_vector_cache_lock = threading.Lock()
+
+
+def _cached_vectors(identity: Optional[str], texts: list[str]) -> list:
+ """The cached vector per text, None for a miss. Keyed by content, so an edited turn simply misses."""
+ if not identity:
+ return [None] * len(texts)
+ with _vector_cache_lock:
+ out = []
+ for text in texts:
+ key = _chunk_key(identity, text)
+ vector = _vector_cache.get(key)
+ if vector is not None:
+ _vector_cache.move_to_end(key)
+ out.append(vector)
+ return out
+
+
+def _remember_vectors(identity: str, texts: list[str], vectors) -> None:
+ with _vector_cache_lock:
+ for text, vector in zip(texts, vectors):
+ key = _chunk_key(identity, text)
+ # A row of an (N, dim) batch is a view: cached as is, it would keep the whole batch alive.
+ copy = getattr(vector, "copy", None)
+ _vector_cache[key] = copy() if copy is not None else vector
+ _vector_cache.move_to_end(key)
+ while len(_vector_cache) > _VECTOR_CACHE_MAX:
+ _vector_cache.popitem(last = False)
+
+
+def _turn_chunks(text: str, chunk_limit: int, count) -> list:
+ """The exact chunking archive_turns stores, shared so pre-embedded vectors match it."""
+ return chunk_pages(
+ [Page(text = text, page_number = None, char_count = len(text))],
+ max_tokens = chunk_limit,
+ overlap = config.CHUNK_OVERLAP,
+ count = count,
+ )
+
+
+def _embed_pending_chunks(
+ texts: list[str],
+ *,
+ model_name: str,
+ cancel_check: Optional[Callable[[], bool]] = None,
+ expected_identity: Optional[str] = None,
+ cached: Optional[list] = None,
+ max_encode: Optional[int] = None,
+):
+ """``(vectors, identity, cancelled, error)``, partial after a cancel or a late failure:
+ write only groups ``vectors`` fully covers. A failing first batch raises. Only cache misses
+ are encoded."""
+ # `cached` is the lookup the caller's cap was computed from; a second lookup could see a later eviction.
+ vectors = list(cached) if cached is not None else _cached_vectors(expected_identity, texts)
+ identity = expected_identity if any(v is not None for v in vectors) else None
+ misses = [i for i, v in enumerate(vectors) if v is None]
+ restarted = cancelled = False
+ error = None
+ k = 0
+ while k < len(misses):
+ if cancel_check is not None and cancel_check():
+ cancelled = True
+ break
+ idx = misses[k : k + _ARCHIVE_EMBED_BATCH]
+ try:
+ out, batch_identity = embeddings.encode_with_identity(
+ [texts[i] for i in idx], model_name = model_name, normalize = True
+ )
+ except Exception as exc:
+ # Keep finished batches, or a late timeout makes every request redo the whole pass.
+ error = exc
+ break
+ if identity is not None and batch_identity != identity:
+ if restarted:
+ break
+ # An ST failure swaps to llama-server for good: restart in the new space, as ingestion does.
+ logger.warning("conversation_archive.embedder_changed_mid_pass; re-embedding")
+ vectors, identity, restarted, k = [None] * len(texts), None, True, 0
+ # Former cache hits are re-encoded too, so the restart honours the request's cap itself.
+ misses = list(range(len(texts) if max_encode is None else min(len(texts), max_encode)))
+ continue
+ identity = batch_identity
+ for i, vector in zip(idx, out):
+ vectors[i] = vector
+ _remember_vectors(batch_identity, [texts[i] for i in idx], out)
+ k += _ARCHIVE_EMBED_BATCH
+ prefix = []
+ for vector in vectors:
+ if vector is None:
+ break
+ prefix.append(vector)
+ if error is not None and not prefix:
+ raise error
+ return prefix, identity or embeddings.embedding_identity(model_name), cancelled, error
+
+
+_gap = threading.Lock()
+_inference_depth = 0
+_noted_branches: dict = {}
+_prewarm_worker: Optional[threading.Thread] = None
+_prewarm_stop = threading.Event()
+
+
+def note_branch(
+ thread_id: Optional[str],
+ messages: list[dict],
+ cancel_check: Optional[Callable[[], bool]] = None,
+) -> None:
+ """Remember a thread's latest transcript for the next idle gap; replaces the previous snapshot."""
+ if not thread_id or not messages or not enabled():
+ return
+ from utils.account_context import current_account
+
+ # Captured here: the worker is a plain thread, which would otherwise read the owner's databases.
+ account = current_account()
+ with _gap:
+ # A stopped request whose stream never opened would otherwise keep its transcript here indefinitely.
+ for key, job in list(_noted_branches.items()):
+ if job[3] is not None and job[3]():
+ del _noted_branches[key]
+ key = (account.account_id, thread_id)
+ _noted_branches.pop(key, None)
+ while len(_noted_branches) >= _NOTED_MAX:
+ del _noted_branches[next(iter(_noted_branches))]
+ _noted_branches[key] = (
+ account,
+ thread_id,
+ list(messages),
+ cancel_check,
+ )
+
+
+@contextmanager
+def _busy(restart: bool):
+ global _inference_depth
+ with _gap:
+ _inference_depth += 1
+ _pause_prewarm()
+ try:
+ yield
+ finally:
+ with _gap:
+ _inference_depth -= 1
+ if restart:
+ _start_prewarm()
+
+
+def inference_running():
+ """Hold while llama.cpp generates. Stops background embedding and waits out its batch in flight."""
+ return _busy(restart = True)
+
+
+def prewarm_running() -> bool:
+ worker = _prewarm_worker
+ return worker is not None and worker.is_alive()
+
+
+def hold_inference() -> Callable[[], None]:
+ """``inference_running`` for the event loop: never blocks (the batch in flight, at most ``_PREWARM_BATCH``
+ chunks, finishes on its own) and returns the release, to be called exactly once."""
+ global _inference_depth
+ with _gap:
+ _inference_depth += 1
+ _prewarm_stop.set()
+
+ def release() -> None:
+ global _inference_depth
+ with _gap:
+ _inference_depth -= 1
+ _start_prewarm()
+
+ return release
+
+
+def _pause_prewarm() -> None:
+ """Stop the gap worker and wait out its batch in flight; it restarts at the next stream's end. Bounded, so a
+ wedged embedder can never hold a generation back."""
+ _prewarm_stop.set()
+ worker = _prewarm_worker
+ if worker is not None and worker is not threading.current_thread():
+ worker.join(_PAUSE_JOIN_S)
+ if worker.is_alive():
+ logger.warning("conversation_archive.prewarm_pause_timed_out after=%ss", _PAUSE_JOIN_S)
+
+
+def _start_prewarm() -> None:
+ global _prewarm_worker
+ with _gap:
+ if _inference_depth or not _noted_branches:
+ return
+ if _prewarm_worker is not None and _prewarm_worker.is_alive():
+ return
+ jobs = list(_noted_branches.values())
+ _noted_branches.clear()
+ _prewarm_stop.clear()
+ _prewarm_worker = threading.Thread(
+ target = _prewarm_jobs, args = (jobs,), daemon = True, name = "archive-prewarm"
+ )
+ _prewarm_worker.start()
+
+
+def _other_work_busy() -> bool:
+ """Image / video generation or a training run holds the accelerator. Read only from modules already
+ imported (never constructs a service), and an unreadable probe counts as busy."""
+ import sys
+
+ try:
+ for module, attr, probe in (
+ ("core.training.training", "_training_backend", "is_training_active"),
+ ("core.training.diffusion_training_service", "_service", "is_active"),
+ ):
+ service = getattr(sys.modules.get(module), attr, None)
+ if service is not None and getattr(service, probe)():
+ return True
+ residency = sys.modules.get("core.inference.gpu_residency")
+ if residency is not None:
+ for owner in residency.MEDIA_OWNERS:
+ if any(residency._media_busy(owner)):
+ return True
+ except Exception: # noqa: BLE001 -- fail closed: skipping a warm-up costs nothing
+ return True
+ return False
+
+
+def _gap_closed() -> bool:
+ return _prewarm_stop.is_set() or _inference_depth > 0 or _other_work_busy()
+
+
+def _requeue(jobs: list) -> None:
+ """Put back what a closed gap interrupted, behind nothing newer: a fresh note for the same chat wins, and a
+ stopped request stays dropped."""
+ with _gap:
+ kept = {
+ (job[0].account_id, job[1]): job
+ for job in jobs
+ if (job[0].account_id, job[1]) not in _noted_branches and not (job[3] and job[3]())
+ }
+ merged = {**kept, **_noted_branches}
+ _noted_branches.clear()
+ _noted_branches.update(list(merged.items())[-_NOTED_MAX:])
+
+
+def _prewarm_jobs(jobs: list) -> None:
+ from utils.account_context import run_as
+ for index, (account, thread_id, messages, cancel_check) in enumerate(jobs):
+ if _gap_closed():
+ _requeue(jobs[index:])
+ return
+ # Chat Stop ends this request's background work too, not only its foreground pass.
+ stopped = cancel_check or (lambda: False)
+ if stopped():
+ continue
+ try:
+ run_as(account, _prewarm_thread, thread_id, messages, stopped)
+ except Exception: # noqa: BLE001 -- a cache warm-up is never worth a log storm
+ logger.debug(
+ "conversation_archive.prewarm_failed thread_id=%s", thread_id, exc_info = True
+ )
+ if _gap_closed():
+ # Possibly cut short; cached vectors make the redo cheap.
+ _requeue(jobs[index:])
+ return
+
+
+def _prewarm_thread(
+ thread_id: str,
+ messages: list[dict],
+ stopped: Callable[[], bool] = lambda: False,
+) -> None:
+ """Embed this thread's not-yet-archived turns into the vector cache, oldest first (the next to be
+ evicted). Writes nothing to the archive, so an edit or rewind cannot strand a stale row."""
+ if not can_archive(thread_id):
+ return
+ from core.inference.context_window import group_turns
+
+ model = config.effective_embedding_model()
+ if not _embedder_free():
+ return
+ identity = embeddings.embedding_identity(model)
+ scope = store.conversation_archive_scope(thread_id)
+ count = _yielding(embeddings.token_counter(model))
+ chunk_limit = embeddings.chunk_tokens(model)
+ conn = rag_db.get_connection()
+ try:
+ budget, todo, done = _PREWARM_MAX_CHUNKS, [], 0
+ for group in group_turns(messages):
+ if _gap_closed() or stopped() or budget <= 0:
+ break
+ archivable = _archivable(group)
+ text = render_turn(archivable) if archivable else ""
+ if not text:
+ continue
+ digest = hashlib.sha256(text.encode("utf-8", "ignore")).hexdigest()
+ if _archived_under(conn, scope, digest, identity):
+ continue
+ texts = [chunk.text for chunk in _turn_chunks(text, chunk_limit, count)]
+ misses = [t for t, v in zip(texts, _cached_vectors(identity, texts)) if v is None]
+ todo += misses[:budget]
+ budget -= len(misses)
+ while len(todo) >= _PREWARM_BATCH and not (_gap_closed() or stopped()):
+ done += _prewarm_batch(model, identity, todo[:_PREWARM_BATCH])
+ todo = todo[_PREWARM_BATCH:]
+ while todo and not (_gap_closed() or stopped()):
+ done += _prewarm_batch(model, identity, todo[:_PREWARM_BATCH])
+ todo = todo[_PREWARM_BATCH:]
+ except _GapClosed:
+ pass
+ finally:
+ conn.close()
+ if done:
+ logger.info("conversation_archive.prewarmed thread_id=%s chunks=%s", thread_id, done)
+
+
+class _GapClosed(Exception):
+ pass
+
+
+def _yielding(count: Callable[[str], int]) -> Callable[[str], int]:
+ def call(text: str) -> int:
+ if not _embedder_free():
+ raise _GapClosed
+ return count(text)
+
+ return call
+
+
+def _embedder_busy() -> bool:
+ """Whether another caller holds a lock the encode path would wait on: model load / backend build, the ST
+ compute lock, or the llama-server backend's serve lock. Probed without blocking."""
+ locks = [embeddings._backend_lock, embeddings._lock, embeddings._compute_lock]
+ serve = getattr(embeddings._backend, "_serve_lock", None)
+ if serve is not None:
+ locks.append(serve)
+ for lock in locks:
+ if not lock.acquire(blocking = False):
+ return True
+ lock.release()
+ return False
+
+
+def _embedder_free() -> bool:
+ """Wait for the embedder without queueing on its locks: a queued worker cannot see a stop, so a generation
+ would wait behind another caller's (e.g. ingestion's) whole batch."""
+ while _embedder_busy():
+ if _gap_closed():
+ return False
+ time.sleep(0.05)
+ return not _gap_closed()
+
+
+def _prewarm_batch(model: str, identity: str, texts: list[str]) -> int:
+ if not _embedder_free():
+ return 0
+ vectors, got = embeddings.encode_with_identity(texts, model_name = model, normalize = True)
+ if got != identity:
+ # The embedder changed under us; the next request's archive re-embeds in the new space.
+ _prewarm_stop.set()
+ return 0
+ _remember_vectors(got, texts, vectors)
+ return len(texts)
+
+
def archive_turns(
thread_id: str,
evicted: list[dict],
live: Optional[list[dict]] = None,
branch: Optional[list[dict]] = None,
+ cancel_check: Optional[Callable[[], bool]] = None,
) -> int:
"""Index the evicted turns for ``thread_id``. Returns how many were newly written.
@@ -373,6 +767,8 @@ def archive_turns(
``live`` is the fitted conversation, i.e. what the model can still see. It bounds how many
copies of a repeated turn the archive is allowed to hold right now; see ``_write_budget``.
Optional, and omitting it keeps the previous behaviour.
+
+ ``cancel_check`` (Chat Stop) is polled before chunking and between embedding batches.
"""
if not thread_id or not evicted or not enabled():
return 0
@@ -381,7 +777,19 @@ def archive_turns(
if not can_archive(thread_id):
logger.debug("conversation_archive.skipped_unpersisted_thread thread_id=%s", thread_id)
return 0
+ # A request's own compaction goes first and stays marked busy: another slot's stream ending must not
+ # restart the gap worker under it. Only a stream's end restarts it, so none starts before this stream.
+ with _busy(restart = False):
+ return _archive_turns(thread_id, evicted, live, branch, cancel_check)
+
+def _archive_turns(
+ thread_id: str,
+ evicted: list[dict],
+ live: Optional[list[dict]],
+ branch: Optional[list[dict]],
+ cancel_check: Optional[Callable[[], bool]],
+) -> int:
# The evictor's own grouper, so an archived unit is exactly the unit the window drops. Imported lazily because the
# inference layer imports this module.
from core.inference.context_window import group_turns
@@ -407,6 +815,8 @@ def archive_turns(
written = 0
conn = None
try:
+ if cancel_check is not None and cancel_check():
+ return 0
count = embeddings.token_counter(model)
chunk_limit = None
conn = rag_db.get_connection()
@@ -416,8 +826,12 @@ def archive_turns(
# Chunk everything first, then embed it in ONE pass. Per group, a first compaction of a long chat ran dozens
# of one-item embedding jobs back to back before the reply could start, and both backends serialise them.
- pending = []
+ pending, needs, snapshots = [], [], []
+ max_chunks = _archive_embed_max_chunks()
+ collected = skipped = 0
for group, span in groups:
+ if cancel_check is not None and cancel_check():
+ return 0
text = render_turn(group)
if not text:
continue
@@ -442,30 +856,108 @@ def archive_turns(
)
_widen_span(conn, scope, digest, span)
continue
+ if collected >= max_chunks:
+ skipped += 1
+ continue
if chunk_limit is None:
chunk_limit = embeddings.chunk_tokens(model)
- chunks = chunk_pages(
- [Page(text = text, page_number = None, char_count = len(text))],
- max_tokens = chunk_limit,
- overlap = config.CHUNK_OVERLAP,
- count = count,
- )
+ chunks = _turn_chunks(text, chunk_limit, count)
if chunks:
+ # The cap counts what must be encoded now; chunks pre-embedded in an idle gap are free.
+ hits = _cached_vectors(expected_identity, [chunk.text for chunk in chunks])
+ need = sum(v is None for v in hits)
pending.append((group, span, digest, chunks, seats, budget))
+ needs.append(need)
+ snapshots.append(hits)
+ collected += need
if not pending:
return 0
+ if cancel_check is not None and cancel_check():
+ logger.info(
+ "conversation_archive.embed_cancelled_before_encode thread_id=%s groups=%s",
+ thread_id,
+ len(pending),
+ )
+ return 0
+
+ chunk_total = sum(needs)
+ if chunk_total > max_chunks or skipped:
+ kept, used = [], 0
+ for entry, need in zip(pending, needs):
+ # On what was kept, not on chunk count: a fully cached first turn adds nothing to `used`.
+ if kept and used + need > max_chunks:
+ break
+ if need > max_chunks and not kept:
+ # An oversized first turn still goes, or it would block the backlog forever.
+ kept.append(entry)
+ used += need
+ break
+ kept.append(entry)
+ used += need
+ deferred = len(pending) - len(kept) + skipped
+ logger.info(
+ "conversation_archive.embed_capped thread_id=%s groups=%s chunks=%s "
+ "cap=%s deferred_groups=%s",
+ thread_id,
+ len(pending),
+ chunk_total,
+ max_chunks,
+ deferred,
+ )
+ snapshots = snapshots[: len(kept)]
+ pending = kept
+ if not pending:
+ return 0
+
+ texts = [chunk.text for entry in pending for chunk in entry[3]]
+ started = time.monotonic()
+ logger.info(
+ "conversation_archive.embed_start thread_id=%s groups=%s chunks=%s batch=%s",
+ thread_id,
+ len(pending),
+ len(texts),
+ _ARCHIVE_EMBED_BATCH,
+ )
# Identity from the encode that produced these vectors: a concurrent embedder swap would otherwise label them
# with a space they were never in.
- vectors, identity = embeddings.encode_with_identity(
- [chunk.text for entry in pending for chunk in entry[3]],
+ vectors, identity, cancelled, embed_error = _embed_pending_chunks(
+ texts,
model_name = model,
- normalize = True,
+ cancel_check = cancel_check,
+ expected_identity = expected_identity,
+ cached = [vector for hits in snapshots for vector in hits],
+ max_encode = max(max_chunks, sum(needs[: len(pending)])),
)
+ logger.info(
+ "conversation_archive.embed_done thread_id=%s vectors=%s/%s cancelled=%s elapsed_ms=%s",
+ thread_id,
+ len(vectors),
+ len(texts),
+ cancelled,
+ int((time.monotonic() - started) * 1000),
+ )
+ if not vectors:
+ return 0
+
+ # Only fully embedded groups; a partial one is left for the next request.
+ complete: list = []
offset = 0
- for group, span, digest, chunks, seats, budget in pending:
- group_vectors = vectors[offset : offset + len(chunks)]
- offset += len(chunks)
+ for entry in pending:
+ n = len(entry[3])
+ if offset + n > len(vectors):
+ break
+ complete.append((entry, vectors[offset : offset + n]))
+ offset += n
+ if len(complete) < len(pending):
+ logger.info(
+ "conversation_archive.embed_partial thread_id=%s written_groups=%s/%s",
+ thread_id,
+ len(complete),
+ len(pending),
+ )
+
+ for (group, span, digest, chunks, seats, budget), group_vectors in complete:
roles = " + ".join(
dict.fromkeys(str(message.get("role") or "message") for message in group)
)
@@ -591,6 +1083,8 @@ def archive_turns(
written += 1
if _INGEST_FAILED:
globals()["_INGEST_FAILED"] = False
+ if embed_error is not None:
+ raise embed_error
except embeddings.EmbeddingModelDownloadRequiredError as exc:
globals()["_INGEST_FAILED"] = True
_log_embedder_pending(model, exc)
diff --git a/studio/backend/core/training/diffusion_training_service.py b/studio/backend/core/training/diffusion_training_service.py
index 9db37a7e557..8dbddc39b26 100644
--- a/studio/backend/core/training/diffusion_training_service.py
+++ b/studio/backend/core/training/diffusion_training_service.py
@@ -17,6 +17,7 @@ runs a scripted target on a thread.
from __future__ import annotations
+from core.inference.inference_gate import pauses_background_embedding
from core.training.account_jobs import (
account_is_retired,
init_job_owner,
@@ -665,6 +666,7 @@ class DiffusionTrainingService:
@_invalidates_gpu_memory("diffusion training start")
@owned_job()
+ @pauses_background_embedding
def start(self, config: dict) -> str:
"""Validate ``config``, spawn the trainer, and start pumping its events.
diff --git a/studio/backend/core/training/training.py b/studio/backend/core/training/training.py
index 8e95901766a..ef0898824eb 100644
--- a/studio/backend/core/training/training.py
+++ b/studio/backend/core/training/training.py
@@ -10,6 +10,7 @@ orchestrates the subprocess lifecycle, pumps events from the worker's mp.Queue,
exposes the same API to routes/training.py. Pattern follows data_recipe/jobs/manager.py.
"""
+from core.inference.inference_gate import pauses_background_embedding
from core.training.account_jobs import (
account_is_retired,
account_process_spec,
@@ -1711,6 +1712,7 @@ class TrainingBackend:
@_invalidates_gpu_memory("training start")
@owned_job()
+ @pauses_background_embedding
def start_training(
self,
job_id: str,
diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py
index 5f6817e7dce..5b482ec4895 100644
--- a/studio/backend/routes/inference.py
+++ b/studio/backend/routes/inference.py
@@ -150,6 +150,7 @@ from core.inference.orchestrator import (
_summed_tool_loop_stats,
routed_slot,
)
+from core.inference.inference_gate import gated_async_client, holds_inference
from core.inference.llama_admission import (
LlamaAdmissionCancelled,
LlamaAdmissionConfig,
@@ -5986,7 +5987,7 @@ async def _await_stt_disconnect_then_cancel(request, sidecar, cancel_event) -> N
def _cancelable_nonstreaming_client() -> httpx.AsyncClient:
- return httpx.AsyncClient(
+ return gated_async_client(
limits = httpx.Limits(max_connections = 1, max_keepalive_connections = 0),
trust_env = False,
verify = _local_ssl_context(),
@@ -8335,11 +8336,22 @@ def _direct_llama_request(counted: bool = True):
if not counted:
yield
return
+ from core.rag.conversation_archive import _pause_prewarm, hold_inference
+
+ # Inference too (caption / OCR, TTS): background archive embedding stays paused. A worker thread (caption /
+ # OCR) also waits out the batch in flight; the TTS route enters this on the event loop, and its generation
+ # runs the gated generate_audio_response in a thread, which waits there.
+ release = hold_inference()
+ try:
+ asyncio.get_running_loop()
+ except RuntimeError:
+ _pause_prewarm()
_direct_llama_request_started()
try:
yield
finally:
_direct_llama_request_finished()
+ release()
def _direct_llama_is_busy() -> bool:
@@ -27461,6 +27473,7 @@ async def _transcribe_audio_bytes(
)
+@holds_inference
async def _transcribe_audio_result(
raw: bytes,
model: Optional[str],
@@ -27542,12 +27555,16 @@ async def _transcribe_audio_result(
)
load_options = {"on_phase": on_phase} if on_phase is not None else {}
if source_path is not None and timestamps:
- await asyncio.to_thread(sidecar.ensure_aligner, model, on_phase, cancel_event)
+ await asyncio.to_thread(
+ holds_inference(sidecar.ensure_aligner), model, on_phase, cancel_event
+ )
# Started with its aligner now, or transcribe_path would restart it.
load_options["timestamps"] = True
await asyncio.to_thread(
- functools.partial(
- load_stt, model, serving_engine, cancel_event, device = device, **load_options
+ holds_inference(
+ functools.partial(
+ load_stt, model, serving_engine, cancel_event, device = device, **load_options
+ )
)
)
loaded = getattr(sidecar, "loaded_model", None)
@@ -27555,19 +27572,21 @@ async def _transcribe_audio_result(
account_access.note_resident_account(f"stt:{serving_engine}", loaded)
if source_path is not None:
result = await asyncio.to_thread(
- functools.partial(
- sidecar.transcribe_path,
- source_path,
- model,
- language,
- timestamps = timestamps,
- cancel_event = cancel_event,
- on_phase = on_phase,
+ holds_inference(
+ functools.partial(
+ sidecar.transcribe_path,
+ source_path,
+ model,
+ language,
+ timestamps = timestamps,
+ cancel_event = cancel_event,
+ on_phase = on_phase,
+ )
)
)
elif on_progress is not None and serving_engine in ("transformers", "mtmd"):
result = await asyncio.to_thread(
- transcribe,
+ holds_inference(transcribe),
raw,
model,
language,
@@ -27576,10 +27595,12 @@ async def _transcribe_audio_result(
on_progress = on_progress,
)
elif cancel_event is None:
- result = await asyncio.to_thread(transcribe, raw, model, language, fast)
+ result = await asyncio.to_thread(
+ holds_inference(transcribe), raw, model, language, fast
+ )
else:
result = await asyncio.to_thread(
- transcribe,
+ holds_inference(transcribe),
raw,
model,
language,
@@ -40384,7 +40405,7 @@ async def openai_completions(request: Request, current_subject: str = Depends(ge
from core.inference.llama_cpp import LlamaCppBackend
from core.inference.llama_keepwarm import mark_response_failed
- client = httpx.AsyncClient(
+ client = gated_async_client(
timeout = _llama_streaming_generation_timeout(),
trust_env = False,
verify = _local_ssl_context(),
@@ -43614,7 +43635,7 @@ async def _responses_stream(
_relay_recording_tool_extras(upstream.body_iterator, current_subject)
)
else:
- client = httpx.AsyncClient(
+ client = gated_async_client(
timeout = _llama_streaming_generation_timeout(),
trust_env = False,
verify = _local_ssl_context(),
@@ -48797,7 +48818,7 @@ async def _anthropic_passthrough_stream(
# finalizer has anything orphaned to finalize. Each aclose is wrapped in
# `try: ... except Exception: pass` so nested anyio cleanup noise can't
# bubble out.
- client = httpx.AsyncClient(
+ client = gated_async_client(
timeout = _llama_streaming_generation_timeout(),
limits = httpx.Limits(max_keepalive_connections = 0),
trust_env = False,
@@ -50114,7 +50135,7 @@ async def _openai_passthrough_stream_admitted(
# Keep the pre-header window short so accepted SSE clients receive
# immediate headers in the common timeout-reduced stall.
- client = httpx.AsyncClient(
+ client = gated_async_client(
timeout = _llama_streaming_generation_timeout(),
limits = httpx.Limits(max_keepalive_connections = 0),
trust_env = False,
diff --git a/studio/backend/routes/managed_engine_chat.py b/studio/backend/routes/managed_engine_chat.py
index 37d4b5a2cd8..eee446388c9 100644
--- a/studio/backend/routes/managed_engine_chat.py
+++ b/studio/backend/routes/managed_engine_chat.py
@@ -15,6 +15,7 @@ from fastapi import HTTPException
from fastapi.responses import JSONResponse
from core.inference.engine_transport import engine_messages, engine_request_timeout
+from core.inference.inference_gate import gated_async_client
from core.inference.http_stream import closing_response_lines
from core.inference.studio_tool_loop import ToolLoopPolicy, ToolLoopRun, stream_with_studio_tools
@@ -296,7 +297,7 @@ async def managed_tool_chat(
tracker.__enter__()
async def events():
- client = httpx.AsyncClient(trust_env = False, timeout = engine_request_timeout())
+ client = gated_async_client(trust_env = False, timeout = engine_request_timeout())
watcher = asyncio.create_task(
api._await_cancel_or_disconnect_then_close_client(
cancel_event = cancel, request = request, client = client
@@ -526,7 +527,7 @@ async def managed_engine_completions(request, current_subject, backend, body):
cancel = threading.Event()
tracker = api._TrackedCancel(cancel, model = public, kind = "completions")
tracker.__enter__()
- client = httpx.AsyncClient(trust_env = False, timeout = engine_request_timeout())
+ client = gated_async_client(trust_env = False, timeout = engine_request_timeout())
watcher = asyncio.create_task(
api._await_cancel_or_disconnect_then_close_client(
cancel_event = cancel, request = request, client = client
diff --git a/studio/backend/tests/conftest.py b/studio/backend/tests/conftest.py
index 72d97343c5b..ceb98d4014d 100644
--- a/studio/backend/tests/conftest.py
+++ b/studio/backend/tests/conftest.py
@@ -874,6 +874,23 @@ def dit_train_host(monkeypatch):
return dtc
[email protected](autouse = True)
+def _fresh_archive_vector_cache():
+ """The archive's pre-embedded vector cache is process-wide; one test's vectors must not answer another's."""
+ archive = sys.modules.get("core.rag.conversation_archive")
+ if archive is not None:
+ archive._vector_cache.clear()
+ archive._noted_branches.clear()
+ yield
+ archive = sys.modules.get("core.rag.conversation_archive")
+ if archive is not None:
+ archive._prewarm_stop.set()
+ if archive._prewarm_worker is not None:
+ archive._prewarm_worker.join()
+ archive._vector_cache.clear()
+ archive._noted_branches.clear()
+
+
@pytest.fixture(autouse = True)
def _reset_optional_module_memo():
"""_load_optional caches failures too, so one test's fake module would answer the next question."""
diff --git a/studio/backend/tests/test_conversation_archive.py b/studio/backend/tests/test_conversation_archive.py
index 781ec3c60a5..e8b61b67b72 100644
--- a/studio/backend/tests/test_conversation_archive.py
+++ b/studio/backend/tests/test_conversation_archive.py
@@ -1464,6 +1464,883 @@ def test_a_first_compaction_embeds_its_turns_in_one_pass(conn, monkeypatch):
assert calls[0] >= 12
+def test_archive_embed_honors_cancel_between_batches(conn, monkeypatch):
+ """Chat Stop must interrupt the pre-inference SentenceTransformer.encode (#14519)."""
+ from core.rag import embeddings
+
+ thread_id = "cancel-embed-thread"
+ _save_thread(thread_id, _turn("hello", "hi"), append = True)
+ monkeypatch.setattr(conversation_archive, "_ARCHIVE_EMBED_BATCH", 2)
+ monkeypatch.setattr(conversation_archive, "_ARCHIVE_EMBED_MAX_CHUNKS_DEFAULT", 10_000)
+
+ calls = []
+ real = embeddings.encode_with_identity
+
+ def counted(texts, **kwargs):
+ calls.append(len(texts))
+ return real(texts, **kwargs)
+
+ monkeypatch.setattr(embeddings, "encode_with_identity", counted)
+
+ evicted = []
+ for index in range(8):
+ evicted.append({"role": "user", "content": f"cancel question {index} unique {index}"})
+ evicted.append({"role": "assistant", "content": f"cancel answer {index} unique {index}"})
+
+ def cancel_check():
+ return bool(calls)
+
+ written = conversation_archive.archive_turns(thread_id, evicted, cancel_check = cancel_check)
+ assert 0 < written < 8
+ assert len(calls) == 1
+ assert conversation_archive.archive_turns(thread_id, evicted) == 8 - written
+
+
+def test_archive_embed_caps_chunks_per_call(conn, monkeypatch):
+ """A huge first compaction must not monopolise the generation path (#14519)."""
+ from core.rag import embeddings
+
+ thread_id = "cap-embed-thread"
+ _save_thread(thread_id, _turn("hello", "hi"), append = True)
+ monkeypatch.setenv("RAG_ARCHIVE_EMBED_MAX_CHUNKS", "4")
+ monkeypatch.setattr(conversation_archive, "_ARCHIVE_EMBED_BATCH", 64)
+
+ calls = []
+ real = embeddings.encode_with_identity
+
+ def counted(texts, **kwargs):
+ calls.append(len(texts))
+ return real(texts, **kwargs)
+
+ monkeypatch.setattr(embeddings, "encode_with_identity", counted)
+
+ evicted = []
+ for index in range(10):
+ evicted.append({"role": "user", "content": f"cap question {index} unique token {index}"})
+ evicted.append({"role": "assistant", "content": f"cap answer {index} unique token {index}"})
+
+ first = conversation_archive.archive_turns(thread_id, evicted)
+ assert 0 < first < 10
+ assert calls and calls[0] <= 4
+ total = first
+ for _ in range(10):
+ written = conversation_archive.archive_turns(thread_id, evicted)
+ if not written:
+ break
+ total += written
+ assert total == 10
+ assert all(n <= 4 for n in calls)
+ assert conversation_archive.archive_turns(thread_id, evicted) == 0
+
+
+def test_a_stop_before_chunking_skips_the_embedder_entirely(conn, monkeypatch):
+ """Stop already set: no tokenizer load, no chunking, no encode (#14519)."""
+ from core.rag import embeddings
+
+ thread_id = "stop-early-thread"
+ _save_thread(thread_id, _turn("hello", "hi"), append = True)
+ touched = []
+ monkeypatch.setattr(embeddings, "token_counter", lambda *_a, **_k: touched.append("count"))
+ monkeypatch.setattr(
+ conversation_archive, "chunk_pages", lambda *_a, **_k: touched.append("chunk")
+ )
+ evicted = _turn("early question", "early answer")
+
+ assert conversation_archive.archive_turns(thread_id, evicted, cancel_check = lambda: True) == 0
+ assert touched == []
+
+
+def test_chunking_stops_once_the_cap_is_reached(conn, monkeypatch):
+ """A long backlog must not be tokenized in full when only the cap's worth is embedded."""
+ thread_id = "cap-chunk-thread"
+ _save_thread(thread_id, _turn("hello", "hi"), append = True)
+ monkeypatch.setenv("RAG_ARCHIVE_EMBED_MAX_CHUNKS", "3")
+ real = conversation_archive.chunk_pages
+ chunked = []
+
+ def counted(*args, **kwargs):
+ chunked.append(1)
+ return real(*args, **kwargs)
+
+ monkeypatch.setattr(conversation_archive, "chunk_pages", counted)
+ evicted = []
+ for index in range(20):
+ evicted += _turn(f"chunk question {index} unique {index}", f"chunk answer {index}")
+
+ assert conversation_archive.archive_turns(thread_id, evicted) == 3
+ assert len(chunked) == 3
+
+
+def test_archive_embed_restarts_in_the_new_space_after_a_backend_swap(conn, monkeypatch):
+ """Batches before an ST -> llama-server swap must not be written under the old identity."""
+ from core.rag import embeddings
+
+ thread_id = "swap-embed-thread"
+ _save_thread(thread_id, _turn("hello", "hi"), append = True)
+ monkeypatch.setattr(conversation_archive, "_ARCHIVE_EMBED_BATCH", 2)
+ monkeypatch.setattr(conversation_archive, "_ARCHIVE_EMBED_MAX_CHUNKS_DEFAULT", 10_000)
+
+ real = embeddings.encode_with_identity
+ calls = []
+
+ def swapping(texts, **kwargs):
+ calls.append(len(texts))
+ vectors, _identity = real(texts, **kwargs)
+ return vectors, "sentence-transformers:old" if len(calls) <= 2 else "llama-server:new"
+
+ monkeypatch.setattr(embeddings, "encode_with_identity", swapping)
+
+ evicted = []
+ for index in range(8):
+ evicted.append({"role": "user", "content": f"swap question {index} unique {index}"})
+ evicted.append({"role": "assistant", "content": f"swap answer {index} unique {index}"})
+
+ assert conversation_archive.archive_turns(thread_id, evicted) == 8
+ scope = store.conversation_archive_scope(thread_id)
+ labels = {
+ row[0]
+ for row in conn.execute("SELECT embedding_model FROM documents WHERE scope = ?", (scope,))
+ }
+ assert labels == {"llama-server:new"}
+
+
+def _gap_thread(
+ thread_id,
+ n,
+ tag = "gap",
+):
+ turns = []
+ for index in range(n):
+ turns += _turn(
+ f"{tag} question {index} unique {index}", f"{tag} answer {index} code {index}"
+ )
+ _save_thread(thread_id, turns, append = True)
+ return turns
+
+
+def _counting_encoder(monkeypatch):
+ from core.rag import embeddings
+
+ calls = []
+ real = embeddings.encode_with_identity
+
+ def counted(texts, **kwargs):
+ calls.append(list(texts))
+ return real(texts, **kwargs)
+
+ monkeypatch.setattr(embeddings, "encode_with_identity", counted)
+ return calls
+
+
+def _run_gap():
+ conversation_archive._start_prewarm()
+ worker = conversation_archive._prewarm_worker
+ if worker is not None:
+ worker.join()
+
+
+def test_an_idle_gap_pre_embeds_so_compaction_encodes_nothing(conn, monkeypatch):
+ """Turns embedded while llama.cpp is idle cost no encode when they are evicted (#14519)."""
+ thread_id = "gap-thread"
+ turns = _gap_thread(thread_id, 6)
+ calls = _counting_encoder(monkeypatch)
+
+ conversation_archive.note_branch(thread_id, turns)
+ _run_gap()
+ assert calls
+ scope = store.conversation_archive_scope(thread_id)
+ # The gap only fills a cache: an edit or rewind before the next request strands nothing.
+ assert (
+ conn.execute("SELECT COUNT(*) FROM documents WHERE scope = ?", (scope,)).fetchone()[0] == 0
+ )
+
+ calls.clear()
+ assert conversation_archive.archive_turns(thread_id, turns[:8]) == 4
+ assert calls == []
+
+
+def test_inference_stops_the_gap_worker_and_it_never_overlaps(conn, monkeypatch):
+ """No embedding batch may run while a llama.cpp stream is open."""
+ import threading
+ import time
+
+ from core.rag import embeddings
+
+ thread_id = "overlap-thread"
+ turns = _gap_thread(thread_id, 40, tag = "overlap")
+ monkeypatch.setattr(conversation_archive, "_PREWARM_BATCH", 1)
+ real = embeddings.encode_with_identity
+ started = threading.Event()
+ overlapped = []
+
+ def slow(texts, **kwargs):
+ started.set()
+ overlapped.append(conversation_archive._inference_depth > 0)
+ time.sleep(0.02)
+ return real(texts, **kwargs)
+
+ monkeypatch.setattr(embeddings, "encode_with_identity", slow)
+ conversation_archive.note_branch(thread_id, turns)
+ conversation_archive._start_prewarm()
+ assert started.wait(5)
+
+ with conversation_archive.inference_running():
+ worker = conversation_archive._prewarm_worker
+ assert worker is None or not worker.is_alive()
+ before = len(overlapped)
+ time.sleep(0.1)
+ assert len(overlapped) == before
+ assert not any(overlapped)
+ assert len(overlapped) < 40
+
+
+def test_a_foreground_archive_stops_the_gap_worker_first(conn, monkeypatch):
+ """A new request's compaction must not queue behind a background job (it runs before the stream opens)."""
+ import threading
+ import time
+
+ from core.rag import embeddings
+
+ background = "fg-background-thread"
+ turns = _gap_thread(background, 40, tag = "fgbg")
+ foreground = "fg-foreground-thread"
+ fg_turns = _gap_thread(foreground, 2, tag = "fgfg")
+ monkeypatch.setattr(conversation_archive, "_PREWARM_BATCH", 1)
+ real = embeddings.encode_with_identity
+ started = threading.Event()
+ worker_calls = []
+
+ def slow(texts, **kwargs):
+ if threading.current_thread().name == "archive-prewarm":
+ started.set()
+ worker_calls.append(len(texts))
+ time.sleep(0.02)
+ return real(texts, **kwargs)
+
+ monkeypatch.setattr(embeddings, "encode_with_identity", slow)
+ conversation_archive.note_branch(background, turns)
+ conversation_archive._start_prewarm()
+ assert started.wait(5)
+
+ assert conversation_archive.archive_turns(foreground, fg_turns) == 2
+ worker = conversation_archive._prewarm_worker
+ assert worker is None or not worker.is_alive()
+ assert len(worker_calls) < 40
+
+
+def test_another_slots_stream_ending_does_not_restart_the_worker_mid_compaction(conn, monkeypatch):
+ """With several llama.cpp slots, a stream can end while this request is still compacting."""
+ import threading
+
+ from core.rag import embeddings
+
+ thread_id = "slots-thread"
+ turns = _gap_thread(thread_id, 2, tag = "slots")
+ other = _gap_thread("slots-other", 4, tag = "slotsother")
+ real = embeddings.encode_with_identity
+ in_archive, release = threading.Event(), threading.Event()
+
+ def held(texts, **kwargs):
+ if threading.current_thread().name != "archive-prewarm":
+ in_archive.set()
+ release.wait(5)
+ return real(texts, **kwargs)
+
+ monkeypatch.setattr(embeddings, "encode_with_identity", held)
+ fg = threading.Thread(target = conversation_archive.archive_turns, args = (thread_id, turns))
+ fg.start()
+ assert in_archive.wait(5)
+ conversation_archive.note_branch("slots-other", other)
+ with conversation_archive.inference_running():
+ pass
+ worker = conversation_archive._prewarm_worker
+ assert worker is None or not worker.is_alive()
+ release.set()
+ fg.join(5)
+
+
+def test_every_inference_entry_point_holds_the_gap_closed(conn, monkeypatch):
+ """MLX / Transformers generation, audio and model loads pause the worker too, not only llama streams."""
+ import threading
+
+ from core.inference.diffusion import DiffusionBackend
+ from core.inference.inference_gate import holds_inference, pauses_background_embedding
+ from core.inference.llama_cpp import LlamaCppBackend
+ from core.inference.mlx_diffusion_backend import MlxDiffusionBackend
+ from core.inference.sd_cpp_backend import SdCppDiffusionBackend
+ from core.inference.orchestrator import InferenceOrchestrator
+ from core.inference.video import VideoBackend
+ from core.training.diffusion_training_service import DiffusionTrainingService
+ from core.training.training import TrainingBackend
+ from core.inference import stt_registry
+ from routes import inference as inference_routes
+
+ for fn in (
+ InferenceOrchestrator._generate_dispatched,
+ InferenceOrchestrator._generate_inner,
+ InferenceOrchestrator.generate_audio_response,
+ InferenceOrchestrator.generate_whisper_response,
+ InferenceOrchestrator.generate_audio_input_response,
+ InferenceOrchestrator.load_model,
+ LlamaCppBackend.generate_audio_response,
+ LlamaCppBackend.load_model,
+ DiffusionBackend.generate,
+ SdCppDiffusionBackend.generate,
+ MlxDiffusionBackend.generate,
+ VideoBackend.generate,
+ DiffusionBackend.begin_load,
+ SdCppDiffusionBackend.begin_load,
+ MlxDiffusionBackend.begin_load,
+ VideoBackend.begin_load,
+ inference_routes._transcribe_audio_result,
+ InferenceOrchestrator.load_stt_model,
+ stt_registry.load,
+ TrainingBackend.start_training,
+ DiffusionTrainingService.start,
+ ):
+ assert hasattr(fn, "__wrapped__"), fn.__qualname__
+
+ thread_id = "gate-thread"
+ turns = _gap_thread(thread_id, 30, tag = "gate")
+ monkeypatch.setattr(conversation_archive, "_PREWARM_BATCH", 1)
+ calls = _counting_encoder(monkeypatch)
+ conversation_archive.note_branch(thread_id, turns)
+ conversation_archive._start_prewarm()
+ seen = {}
+
+ @holds_inference
+ def tokens():
+ seen["depth"] = conversation_archive._inference_depth
+ worker = conversation_archive._prewarm_worker
+ seen["worker_alive"] = worker is not None and worker.is_alive()
+ before = len(calls)
+ yield 1
+ seen["embedded_during"] = len(calls) - before
+
+ @holds_inference
+ def load():
+ return conversation_archive._inference_depth
+
+ assert list(tokens()) == [1]
+ assert seen == {"depth": 1, "worker_alive": False, "embedded_during": 0}
+ assert load() == 1
+ assert conversation_archive._inference_depth == 0
+
+ # Async entry points (speech to text) hold it too, waiting out the batch off the event loop.
+ import asyncio
+
+ conversation_archive._start_prewarm()
+
+ @holds_inference
+ async def transcribe():
+ worker = conversation_archive._prewarm_worker
+ return conversation_archive._inference_depth, worker is not None and worker.is_alive()
+
+ assert asyncio.run(transcribe()) == (1, False)
+ assert conversation_archive._inference_depth == 0
+ assert threading.current_thread().name != "archive-prewarm"
+
+ # A training start stops the worker before admitting the job; its busy probe keeps it idle after.
+ conversation_archive._start_prewarm()
+
+ @pauses_background_embedding
+ def start():
+ worker = conversation_archive._prewarm_worker
+ return worker is not None and worker.is_alive()
+
+ assert start() is False
+
+
+def test_an_interrupted_gap_requeues_what_it_did_not_finish(conn, monkeypatch):
+ """A closed gap keeps unfinished chats queued for the next one; a stopped request is still dropped."""
+ import threading
+ import time
+
+ from core.rag import embeddings
+
+ monkeypatch.setattr(conversation_archive, "_PREWARM_BATCH", 1)
+ real = embeddings.encode_with_identity
+ started = threading.Event()
+
+ def slow(texts, **kwargs):
+ started.set()
+ time.sleep(0.02)
+ return real(texts, **kwargs)
+
+ monkeypatch.setattr(embeddings, "encode_with_identity", slow)
+ stop = threading.Event()
+ for name in ("requeue-a", "requeue-b", "requeue-stopped"):
+ cancel = stop.is_set if name.endswith("stopped") else None
+ conversation_archive.note_branch(name, _gap_thread(name, 20, tag = name), cancel)
+ conversation_archive._start_prewarm()
+ assert started.wait(5)
+ stop.set()
+ with conversation_archive.inference_running():
+ queued = [key[1] for key in conversation_archive._noted_branches]
+ # Nothing left for the stream's end to restart, so teardown has no worker to wait out.
+ conversation_archive._noted_branches.clear()
+ assert queued == ["requeue-a", "requeue-b"]
+
+
+def test_a_direct_llama_call_holds_the_gap_closed(conn, monkeypatch):
+ """Caption / OCR during ingestion and TTS post to llama-server directly, outside the chat paths."""
+ import threading
+ import time
+
+ from core.rag import embeddings
+ from routes import inference as inference_routes
+
+ # From a worker thread (caption / OCR) the request waits out a batch already in flight.
+ real = embeddings.encode_with_identity
+ in_batch, encoding = threading.Event(), threading.Event()
+
+ def slow(texts, **kwargs):
+ encoding.set()
+ try:
+ in_batch.set()
+ time.sleep(3)
+ return real(texts, **kwargs)
+ finally:
+ encoding.clear()
+
+ monkeypatch.setattr(embeddings, "encode_with_identity", slow)
+ monkeypatch.setattr(conversation_archive, "_PREWARM_BATCH", 1)
+ conversation_archive.note_branch("direct-thread", _gap_thread("direct-thread", 4, tag = "direct"))
+ conversation_archive._start_prewarm()
+ assert in_batch.wait(5)
+ with inference_routes._direct_llama_request():
+ assert not encoding.is_set()
+ conversation_archive._noted_branches.clear()
+
+ with inference_routes._direct_llama_request():
+ assert conversation_archive._inference_depth == 1
+ assert conversation_archive._prewarm_stop.is_set()
+ assert conversation_archive._inference_depth == 0
+ with inference_routes._direct_llama_request(counted = False):
+ assert conversation_archive._inference_depth == 0
+
+
+def test_cached_vectors_do_not_keep_their_batch_alive(conn):
+ import numpy as np
+
+ batch = np.ones((64, 8), dtype = np.float32)
+ conversation_archive._remember_vectors("identity", [f"chunk {i}" for i in range(64)], batch)
+ [vector] = conversation_archive._cached_vectors("identity", ["chunk 3"])
+ assert vector.base is None and np.array_equal(vector, batch[3])
+
+
+def test_a_job_start_keeps_the_gap_closed_until_it_returns(conn, monkeypatch):
+ """A stream ending while a load validates (before it publishes its busy state) must not restart the worker."""
+ from core.inference.inference_gate import pauses_background_embedding
+
+ thread_id = "start-thread"
+ conversation_archive.note_branch(thread_id, _gap_thread(thread_id, 3, tag = "start"))
+ calls = _counting_encoder(monkeypatch)
+
+ @pauses_background_embedding
+ def begin_load():
+ with conversation_archive.inference_running():
+ pass
+ worker = conversation_archive._prewarm_worker
+ return worker is not None and worker.is_alive()
+
+ assert begin_load() is False
+ assert calls == []
+
+
+def test_a_cancelled_transcription_holds_the_gap_until_its_thread_ends(conn, monkeypatch):
+ """Cancelling the request task returns at once; the speech thread it started keeps running."""
+ import asyncio
+ import threading
+
+ from core.inference.inference_gate import holds_inference
+
+ started, finish = threading.Event(), threading.Event()
+
+ def transcribe():
+ started.set()
+ finish.wait(5)
+
+ async def route():
+ task = asyncio.ensure_future(asyncio.to_thread(holds_inference(transcribe)))
+ await asyncio.to_thread(started.wait, 5)
+ task.cancel()
+ try:
+ await task
+ except asyncio.CancelledError:
+ pass
+ held = conversation_archive._inference_depth
+ finish.set()
+ return held
+
+ # asyncio.run joins the executor on exit, so the thread has ended by then.
+ assert asyncio.run(route()) == 1
+ assert conversation_archive._inference_depth == 0
+
+
+def test_api_passthrough_clients_hold_the_gap_closed_until_closed(conn, monkeypatch):
+ """OpenAI / Anthropic / managed-engine passthroughs talk to the engine over their own per-request client."""
+ import asyncio
+ import gc
+
+ import httpx
+
+ from core.inference import inference_gate
+
+ closed = []
+
+ class FakeClient:
+ def __init__(self, **kwargs):
+ self.kwargs = kwargs
+
+ async def aclose(self):
+ closed.append(self.kwargs)
+
+ # Passthrough tests fake the client class on the httpx module; the gate must build through it.
+ monkeypatch.setattr(httpx, "AsyncClient", FakeClient)
+ thread_id = "passthrough-thread"
+ conversation_archive.note_branch(thread_id, _gap_thread(thread_id, 3, tag = "pass"))
+ calls = _counting_encoder(monkeypatch)
+
+ client = inference_gate.gated_async_client(trust_env = False)
+ assert isinstance(client, FakeClient)
+ assert (
+ conversation_archive._inference_depth == 1 and conversation_archive._prewarm_stop.is_set()
+ )
+ conversation_archive._start_prewarm()
+ worker = conversation_archive._prewarm_worker
+ assert (worker is None or not worker.is_alive()) and calls == []
+
+ asyncio.run(client.aclose())
+ asyncio.run(client.aclose())
+ assert closed == [{"trust_env": False}] * 2
+ assert conversation_archive._inference_depth == 0
+
+ # The first send waits out a batch already in flight, then goes out.
+ import threading
+ import time
+
+ from core.rag import embeddings
+
+ real = embeddings.encode_with_identity
+ in_batch, encoding, sent_mid_batch = threading.Event(), threading.Event(), []
+
+ def slow(texts, **kwargs):
+ encoding.set()
+ try:
+ in_batch.set()
+ # Long enough to still be running when the request goes out on a loaded host.
+ time.sleep(3)
+ return real(texts, **kwargs)
+ finally:
+ encoding.clear()
+
+ class SendingClient(FakeClient):
+ async def send(self, request):
+ sent_mid_batch.append(encoding.is_set())
+ return "response"
+
+ monkeypatch.setattr(httpx, "AsyncClient", SendingClient)
+ monkeypatch.setattr(embeddings, "encode_with_identity", slow)
+ monkeypatch.setattr(conversation_archive, "_PREWARM_BATCH", 1)
+ conversation_archive.note_branch(thread_id, _gap_thread(thread_id, 6, tag = "send"))
+ conversation_archive._start_prewarm()
+ assert in_batch.wait(5)
+ client = inference_gate.gated_async_client()
+ assert asyncio.run(client.send("request")) == "response"
+ assert sent_mid_batch == [False]
+ conversation_archive._noted_branches.clear()
+ asyncio.run(client.aclose())
+ monkeypatch.setattr(httpx, "AsyncClient", FakeClient)
+
+ # A path that never closes its client still reopens the gap once it is collected.
+ inference_gate.gated_async_client()
+ gc.collect()
+ assert conversation_archive._inference_depth == 0
+
+
[email protected]("held", ["compute", "llama_serve", "backend_build"])
+def test_the_gap_worker_never_queues_behind_another_embedder_user(conn, monkeypatch, held):
+ """While ingestion holds the embedder, a new generation must not wait for the worker to get and finish its turn."""
+ import time
+
+ from core.rag import embeddings
+
+ thread_id = "lock-thread"
+ turns = _gap_thread(thread_id, 6, tag = "lock")
+ real = embeddings.encode_with_identity
+
+ import threading
+ import types
+
+ # The llama-server embedder serialises on its own reentrant lock, not the ST one.
+ serve = threading.RLock()
+ monkeypatch.setattr(embeddings, "_backend", types.SimpleNamespace(_serve_lock = serve))
+ lock = {
+ "compute": embeddings._compute_lock,
+ "llama_serve": serve,
+ "backend_build": embeddings._backend_lock,
+ }[held]
+
+ def locked(texts, **kwargs):
+ with lock:
+ time.sleep(0.5)
+ return real(texts, **kwargs)
+
+ monkeypatch.setattr(embeddings, "encode_with_identity", locked)
+ monkeypatch.setattr(conversation_archive, "_PAUSE_JOIN_S", 30.0)
+ conversation_archive.note_branch(thread_id, turns)
+ with lock:
+ conversation_archive._start_prewarm()
+ time.sleep(0.3)
+ started = time.monotonic()
+ with conversation_archive.inference_running():
+ waited = time.monotonic() - started
+ assert waited < 1.0
+
+
[email protected]("busy", ["training", "diffusion_training", "media", "unreadable"])
+def test_image_video_or_training_work_keeps_the_gap_closed(conn, monkeypatch, busy):
+ """Non-LLM accelerator work pauses the worker too; an unreadable probe counts as busy."""
+ import sys
+ import types
+
+ class Service:
+ def is_training_active(self):
+ return busy == "training"
+
+ def is_active(self):
+ return busy == "diffusion_training"
+
+ training = types.SimpleNamespace(_training_backend = Service())
+ diffusion = types.SimpleNamespace(_service = Service())
+
+ def media_busy(owner):
+ if busy == "unreadable":
+ raise RuntimeError("probe failed")
+ return (False, busy == "media")
+
+ residency = types.SimpleNamespace(MEDIA_OWNERS = ("diffusion", "video"), _media_busy = media_busy)
+ monkeypatch.setitem(sys.modules, "core.training.training", training)
+ monkeypatch.setitem(sys.modules, "core.training.diffusion_training_service", diffusion)
+ monkeypatch.setitem(sys.modules, "core.inference.gpu_residency", residency)
+
+ thread_id = f"busy-{busy}"
+ turns = _gap_thread(thread_id, 3, tag = busy)
+ calls = _counting_encoder(monkeypatch)
+ conversation_archive.note_branch(thread_id, turns)
+ _run_gap()
+ assert calls == []
+
+
+def test_the_gap_opens_when_nothing_else_is_running(conn, monkeypatch):
+ import sys
+ import types
+
+ idle = types.SimpleNamespace(is_training_active = lambda: False, is_active = lambda: False)
+ monkeypatch.setitem(
+ sys.modules, "core.training.training", types.SimpleNamespace(_training_backend = idle)
+ )
+ monkeypatch.setitem(
+ sys.modules,
+ "core.training.diffusion_training_service",
+ types.SimpleNamespace(_service = idle),
+ )
+ monkeypatch.setitem(
+ sys.modules,
+ "core.inference.gpu_residency",
+ types.SimpleNamespace(
+ MEDIA_OWNERS = ("diffusion",), _media_busy = lambda owner: (False, False)
+ ),
+ )
+ thread_id = "idle-thread"
+ turns = _gap_thread(thread_id, 3, tag = "idle")
+ calls = _counting_encoder(monkeypatch)
+ conversation_archive.note_branch(thread_id, turns)
+ _run_gap()
+ assert calls
+
+
+def test_an_edited_turn_misses_the_cache_and_is_embedded_fresh(conn, monkeypatch):
+ thread_id = "edit-thread"
+ turns = _gap_thread(thread_id, 2, tag = "edit")
+ calls = _counting_encoder(monkeypatch)
+ conversation_archive.note_branch(thread_id, turns)
+ _run_gap()
+
+ edited = [dict(m) for m in turns]
+ edited[1]["content"] = "edit answer 0 was revised to code 77"
+ _save_thread(thread_id, edited, append = False)
+ calls.clear()
+ assert conversation_archive.archive_turns(thread_id, edited[:2]) == 1
+ assert [text for batch in calls for text in batch] and all(
+ "code 77" in text for batch in calls for text in batch
+ )
+
+
+def test_the_cap_counts_only_chunks_that_still_need_embedding(conn, monkeypatch):
+ thread_id = "cap-cache-thread"
+ turns = _gap_thread(thread_id, 6, tag = "capcache")
+ monkeypatch.setenv("RAG_ARCHIVE_EMBED_MAX_CHUNKS", "2")
+ conversation_archive.note_branch(thread_id, turns)
+ _run_gap()
+
+ calls = _counting_encoder(monkeypatch)
+ assert conversation_archive.archive_turns(thread_id, turns) == 6
+ assert calls == []
+
+
+def test_an_oversized_turn_after_a_cached_one_waits_for_the_next_compaction(conn, monkeypatch):
+ """A pre-embedded first turn counts nothing toward the cap; the huge turn behind it must still be deferred."""
+ thread_id = "cap-cached-then-huge"
+ cached = _gap_thread(thread_id, 1, tag = "cachedfirst")
+ huge = _turn("paste this log", " ".join(f"line{i}" for i in range(6000)))
+ _save_thread(thread_id, huge, append = True)
+ monkeypatch.setenv("RAG_ARCHIVE_EMBED_MAX_CHUNKS", "2")
+ conversation_archive.note_branch(thread_id, cached)
+ _run_gap()
+
+ calls = _counting_encoder(monkeypatch)
+ assert conversation_archive.archive_turns(thread_id, cached + huge) == 1
+ assert calls == []
+
+
+def test_a_gap_job_runs_as_the_account_that_noted_it(monkeypatch):
+ """The worker is a plain thread; without the binding it would read the owner's databases."""
+ from utils.account_context import AccountContext, current_account, run_as
+
+ seen = []
+ monkeypatch.setattr(conversation_archive, "enabled", lambda: True)
+ monkeypatch.setattr(
+ conversation_archive,
+ "_prewarm_thread",
+ lambda thread_id, messages, stopped: seen.append((current_account().account_id, thread_id)),
+ )
+ member = AccountContext("acct-member", "member", "user")
+ run_as(
+ member,
+ conversation_archive.note_branch,
+ "member-thread",
+ [{"role": "user", "content": "x"}],
+ )
+ _run_gap()
+ assert seen == [("acct-member", "member-thread")]
+
+
+def test_a_stopped_request_leaves_no_background_work(conn, monkeypatch):
+ """Stop before the gap: the noted job is dropped, not resumed in the background."""
+ thread_id = "stopped-gap-thread"
+ turns = _gap_thread(thread_id, 4, tag = "stopped")
+ calls = _counting_encoder(monkeypatch)
+ conversation_archive.note_branch(thread_id, turns, lambda: True)
+ _run_gap()
+ assert calls == []
+
+
+def test_stop_during_the_gap_ends_the_background_work(conn, monkeypatch):
+ thread_id = "stop-mid-gap-thread"
+ turns = _gap_thread(thread_id, 20, tag = "midgap")
+ monkeypatch.setattr(conversation_archive, "_PREWARM_BATCH", 1)
+ calls = _counting_encoder(monkeypatch)
+ conversation_archive.note_branch(thread_id, turns, lambda: len(calls) >= 2)
+ _run_gap()
+ assert len(calls) == 2
+
+
+def test_the_cap_holds_when_the_cache_churns_before_the_encode(conn, monkeypatch):
+ """Vectors counted as free stay free even if the cache evicts them before encoding."""
+ thread_id = "churn-thread"
+ turns = _gap_thread(thread_id, 6, tag = "churn")
+ monkeypatch.setenv("RAG_ARCHIVE_EMBED_MAX_CHUNKS", "2")
+ conversation_archive.note_branch(thread_id, turns)
+ _run_gap()
+
+ real = conversation_archive._embed_pending_chunks
+
+ def evict_then_embed(*args, **kwargs):
+ conversation_archive._vector_cache.clear()
+ return real(*args, **kwargs)
+
+ monkeypatch.setattr(conversation_archive, "_embed_pending_chunks", evict_then_embed)
+ calls = _counting_encoder(monkeypatch)
+ assert conversation_archive.archive_turns(thread_id, turns) == 6
+ assert calls == []
+
+
+def test_an_embedder_swap_restart_still_honours_the_cap(conn, monkeypatch):
+ """A restart in the new space re-encodes former cache hits; it must stay within the request's cap."""
+ from core.rag import embeddings
+
+ thread_id = "swap-cap-thread"
+ turns = _gap_thread(thread_id, 6, tag = "swapcap")
+ conversation_archive.note_branch(thread_id, turns)
+ _run_gap()
+ extra = _turn("swapcap fresh question", "swapcap fresh answer")
+ _save_thread(thread_id, extra, append = True)
+ monkeypatch.setenv("RAG_ARCHIVE_EMBED_MAX_CHUNKS", "2")
+
+ real = embeddings.encode_with_identity
+ calls = []
+
+ def swapped(texts, **kwargs):
+ calls.append(len(texts))
+ vectors, _identity = real(texts, **kwargs)
+ return vectors, "llama-server:swapped"
+
+ monkeypatch.setattr(embeddings, "encode_with_identity", swapped)
+ conversation_archive.archive_turns(thread_id, turns + extra)
+ assert calls[0] == 1
+ assert sum(calls[1:]) <= 2
+
+
+def test_noted_snapshots_of_stopped_requests_do_not_pile_up(monkeypatch):
+ monkeypatch.setattr(conversation_archive, "enabled", lambda: True)
+ message = [{"role": "user", "content": "x"}]
+ conversation_archive.note_branch("stopped-a", message, lambda: True)
+ conversation_archive.note_branch("live-b", message)
+ assert [key[1] for key in conversation_archive._noted_branches] == ["live-b"]
+ for index in range(20):
+ conversation_archive.note_branch(f"many-{index}", message)
+ assert len(conversation_archive._noted_branches) <= conversation_archive._NOTED_MAX
+ # Re-noting a queued thread replaces its snapshot without evicting an unrelated chat.
+ queued = list(conversation_archive._noted_branches)
+ conversation_archive.note_branch(queued[-1][1], message)
+ assert set(conversation_archive._noted_branches) == set(queued)
+
+
+def test_archive_embed_failure_keeps_completed_batches(conn, monkeypatch):
+ """A batch failing late in a long pass must not discard the batches before it (#14519)."""
+ from core.rag import embeddings
+
+ thread_id = "fail-embed-thread"
+ _save_thread(thread_id, _turn("hello", "hi"), append = True)
+ monkeypatch.setattr(conversation_archive, "_ARCHIVE_EMBED_BATCH", 2)
+ monkeypatch.setattr(conversation_archive, "_ARCHIVE_EMBED_MAX_CHUNKS_DEFAULT", 10_000)
+
+ real = embeddings.encode_with_identity
+ calls = []
+
+ def flaky(texts, **kwargs):
+ calls.append(len(texts))
+ if len(calls) == 3:
+ raise TimeoutError("embedder timed out")
+ return real(texts, **kwargs)
+
+ monkeypatch.setattr(embeddings, "encode_with_identity", flaky)
+
+ evicted = []
+ for index in range(8):
+ evicted.append({"role": "user", "content": f"fail question {index} unique {index}"})
+ evicted.append({"role": "assistant", "content": f"fail answer {index} unique {index}"})
+
+ written = conversation_archive.archive_turns(thread_id, evicted)
+ assert 0 < written < 8
+ assert conversation_archive.degraded()
+ calls.clear()
+ assert conversation_archive.archive_turns(thread_id, evicted) == 8 - written
+ assert sum(calls) <= 8 - written
+
+
VARIABLE = "ZQXVARA123"
diff --git a/studio/backend/tests/test_conversation_recall_injection.py b/studio/backend/tests/test_conversation_recall_injection.py
index 898ae7dfaf7..9d0d78cac97 100644
--- a/studio/backend/tests/test_conversation_recall_injection.py
+++ b/studio/backend/tests/test_conversation_recall_injection.py
@@ -165,6 +165,28 @@ def test_archive_and_recall_skips_recall_once_already_done(archived):
assert "recalled_chunks" not in result["counts"]
[email protected]("stopped", [False, True])
+def test_a_stopped_request_skips_the_automatic_recall(archived, monkeypatch, stopped):
+ """Recall embeds the query; a Stopped request must not pay for it (#14519)."""
+ recalls = []
+ monkeypatch.setattr(tools_mod, "build_conversation_recall", lambda *_a, **_k: recalls.append(1))
+ before = _conversation() + [{"role": "user", "content": "evicted turn"}]
+ after = _conversation()
+
+ result = llama_cpp._archive_and_recall(
+ after,
+ before,
+ thread_id = THREAD,
+ style = "tool",
+ recall_done = False,
+ recall_budget_tokens = 100_000,
+ cancel_check = lambda: stopped,
+ )
+
+ assert recalls == ([] if stopped else [1])
+ assert result["recalled"] is False
+
+
def test_a_refused_fit_still_archives_what_it_evicted(archived):
"""`recall_done` skips recall without skipping the archive write."""
before = _conversation() + [{"role": "user", "content": "a turn only the fit saw"}]
diff --git a/studio/backend/tests/test_mlx_context_compaction.py b/studio/backend/tests/test_mlx_context_compaction.py
index 6403a8cf451..acdcf38f3ac 100644
--- a/studio/backend/tests/test_mlx_context_compaction.py
+++ b/studio/backend/tests/test_mlx_context_compaction.py
@@ -87,6 +87,19 @@ def test_an_overflowing_prompt_drops_its_oldest_turns_and_reports_the_boundary(
assert result["messages"][-1] is messages[-1]
+def test_chat_stop_reaches_the_archive_embedding_pass(archive_calls):
+ """Stop must be able to interrupt the archive's embedding before inference (#14519)."""
+ import threading
+
+ stop = threading.Event()
+ _fit(_backend(), _thread(), thread_id = "thread-1", cancel_event = stop)
+
+ cancel_check = archive_calls[-1]["cancel_check"]
+ assert cancel_check() is False
+ stop.set()
+ assert cancel_check() is True
+
+
@pytest.mark.parametrize(
"case", ["not_opted_in", "not_mlx", "media", "resumed_reply", "resumed_thought", "no_count"]
)