DeepTutor: Lifelong Personalized Tutoring. https://deeptutor.info/.
Python ai-agentsai-tutorclawbotcli-tooldeepresearchinteractive-learninglearningmulti-agent-systemsrag
fix(knowledge): isolate reindex progress across workers (#1478)
Bingxi Zhao (Frank)b3949a919adc
diff --git a/deeptutor/services/rag/pipelines/llamaindex/embedding_adapter.py b/deeptutor/services/rag/pipelines/llamaindex/embedding_adapter.pyindex c5ca8be7b..07de019f3 100644--- a/deeptutor/services/rag/pipelines/llamaindex/embedding_adapter.py+++ b/deeptutor/services/rag/pipelines/llamaindex/embedding_adapter.py@@ -4,13 +4,15 @@ from __future__ import annotationsimport asynciofrom contextvars import ContextVar+from dataclasses import dataclassimport logging-from typing import Any, List+from typing import Any, Callable, Listfrom llama_index.core import Settingsfrom llama_index.core.base.embeddings.base import BaseEmbeddingfrom llama_index.core.bridge.pydantic import PrivateAttr+from deeptutor.services.config.provider_runtime import EMBEDDING_PROVIDERSfrom deeptutor.services.embedding import EmbeddingConfig, get_embedding_client, get_embedding_configfrom deeptutor.services.embedding.config import scoped_embedding_configfrom deeptutor.services.embedding.validation import validate_embedding_batch@@ -18,6 +20,24 @@ from deeptutor.services.embedding.validation import validate_embedding_batchfrom .config import chunk_geometry+@dataclass+class _IndexingProgress:+ total_batches: int+ provider_batch_size: int+ completed_batches: int = 0+++# An executor thread inherits the indexing operation's context. Keep both the+# callback and its cumulative count there: a timed-out worker must never pick+# up a later task's callback from a shared CustomEmbedding instance (#1478).+_task_progress_callback: ContextVar[Callable[[int, int], None] | None] = ContextVar(+ "llamaindex_progress_callback", default=None+)+_indexing_progress: ContextVar[_IndexingProgress | None] = ContextVar(+ "llamaindex_indexing_progress", default=None+)++def _config_fingerprint(config: EmbeddingConfig) -> tuple[Any, ...]:"""Return the settings fields that affect LlamaIndex embedding behavior."""return (@@ -77,6 +97,29 @@ class CustomEmbedding(BaseEmbedding):"""Set progress callback fn(batch_num, total_batches)."""self._progress_callback = callback+ def __call__(self, nodes, **kwargs):+ """Count provider batches across the full split-node set once."""+ callback = _task_progress_callback.get() or self._progress_callback+ if callback is None or not nodes:+ return super().__call__(nodes, **kwargs)+ config = getattr(self._client, "config", None)+ binding = getattr(config, "binding", "")+ provider = EMBEDDING_PROVIDERS.get(binding)+ provider_limit = provider.max_batch_items if provider else 256+ batch_size = max(1, min(getattr(config, "batch_size", 10), provider_limit))+ outer_size = self.embed_batch_size+ total_batches = sum(+ (min(outer_size, len(nodes) - start) + batch_size - 1) // batch_size+ for start in range(0, len(nodes), outer_size)+ )+ token = _indexing_progress.set(+ _IndexingProgress(total_batches=total_batches, provider_batch_size=batch_size)+ )+ try:+ return super().__call__(nodes, **kwargs)+ finally:+ _indexing_progress.reset(token)+@classmethoddef class_name(cls) -> str:return "custom_embedding"@@ -111,11 +154,25 @@ class CustomEmbedding(BaseEmbedding):async def _aget_text_embeddings(self, texts: List[str]) -> List[List[float]]:client = self.refresh_client()+ callback = _task_progress_callback.get() or self._progress_callback+ progress = _indexing_progress.get()+ if progress is not None and callback is not None:+ offset = progress.completed_batches+ sink = callback++ def report(batch_num: int, _total_batches: int) -> None:+ sink(offset + batch_num, progress.total_batches)++ callback = reportembeddings = await client.embed(texts,- progress_callback=self._progress_callback,+ progress_callback=callback,input_type="search_document",)+ if progress is not None:+ progress.completed_batches += (+ len(texts) + progress.provider_batch_size - 1+ ) // progress.provider_batch_sizereturn validate_embedding_batch(embeddings,expected_count=len(texts),@@ -180,10 +237,8 @@ def configure_llamaindex_settings(logger=None) -> None:def set_progress_callback(callback) -> None:- """Attach an indexing progress callback to the active embedding adapter."""- embed_model = current_embedding()- if isinstance(embed_model, CustomEmbedding):- embed_model.set_progress_callback(callback)+ """Bind a callback to this indexing context, not a shared adapter slot."""+ _task_progress_callback.set(callback)async def verify_embedding_connectivity(logger=None) -> None:diff --git a/deeptutor/services/rag/pipelines/llamaindex/pipeline.py b/deeptutor/services/rag/pipelines/llamaindex/pipeline.pyindex 2b8467e56..a75c4229f 100644--- a/deeptutor/services/rag/pipelines/llamaindex/pipeline.py+++ b/deeptutor/services/rag/pipelines/llamaindex/pipeline.py@@ -7,6 +7,7 @@ from contextvars import copy_contextimport jsonimport loggingfrom pathlib import Path+import threadingimport timeimport tracebackfrom typing import Any, Callable, Dict, List, Optional@@ -39,6 +40,8 @@ DEFAULT_KB_BASE_DIR = str(get_runtime_data_root() / "knowledge_bases")_INDEX_STALL_TIMEOUT_SECONDS = 600.0# How often the stall guard checks the progress heartbeat._INDEX_STALL_POLL_SECONDS = 5.0+_INDEX_WORKERS: dict[str, threading.Event] = {}+_INDEX_WORKERS_LOCK = threading.Lock()SignatureProvider = Callable[[], EmbeddingSignature | None]@@ -47,11 +50,47 @@ class IndexingStallError(RuntimeError):"""Raised when an indexing operation makes no progress for too long."""+def _worker_busy_error() -> IndexingStallError:+ return IndexingStallError(+ "A previous indexing worker for this knowledge base is still running. "+ "Wait for it to finish or restart DeepTutor before retrying."+ )+++def _ensure_index_worker_available(key: str) -> None:+ with _INDEX_WORKERS_LOCK:+ marker = _INDEX_WORKERS.get(key)+ if marker is not None and not marker.is_set():+ raise _worker_busy_error()+++def _claim_index_worker(key: str | None) -> threading.Event | None:+ if key is None:+ return None+ with _INDEX_WORKERS_LOCK:+ previous = _INDEX_WORKERS.get(key)+ if previous is not None and not previous.is_set():+ raise _worker_busy_error()+ marker = threading.Event()+ _INDEX_WORKERS[key] = marker+ return marker+++def _release_index_worker(key: str | None, marker: threading.Event | None) -> None:+ if key is None or marker is None:+ return+ marker.set()+ with _INDEX_WORKERS_LOCK:+ if _INDEX_WORKERS.get(key) is marker:+ _INDEX_WORKERS.pop(key, None)++async def _run_with_stall_guard(fn: Callable[[], Any],*,progress_callback: Optional[Callable[..., Any]] = None,stall_timeout: Optional[float] = None,+ worker_key: str | None = None,) -> Any:"""Run a synchronous indexing step in the executor, failing if it stalls.@@ -65,32 +104,37 @@ async def _run_with_stall_guard(for ``stall_timeout`` seconds.The sync function keeps running in its thread after a stall is raised —- Python cannot interrupt arbitrary synchronous code — but the API call- fails fast with an actionable message instead of waiting forever.-- Bound knowledge-base operations own an embedding adapter in their context,- so concurrent jobs keep independent progress callbacks. Re-arming also- supports legacy callers that still use LlamaIndex's global Settings.+ Python cannot interrupt arbitrary synchronous code. Its callback becomes+ silent and a same-KB retry is rejected until that thread actually exits."""if stall_timeout is None:stall_timeout = _INDEX_STALL_TIMEOUT_SECONDS- # Each run has its own embedding adapter in the operation context. A- # process-global owner would silence healthy concurrent runs. Invalidate- # only this wrapper when its run ends, including on stall while a sync- # worker may still be alive.- active = True+ worker_marker = _claim_index_worker(worker_key)+ progress_live = threading.Event()+ progress_live.set()last_progress = {"at": time.monotonic()}def _heartbeat(*args: Any, **kwargs: Any) -> None:- if not active:+ if not progress_live.is_set():returnlast_progress["at"] = time.monotonic()if progress_callback is not None:progress_callback(*args, **kwargs)+ def _work() -> Any:+ try:+ return fn()+ finally:+ _release_index_worker(worker_key, worker_marker)+set_progress_callback(_heartbeat)- future = asyncio.get_running_loop().run_in_executor(None, copy_context().run, fn)+ try:+ future = asyncio.get_running_loop().run_in_executor(None, copy_context().run, _work)+ except BaseException:+ progress_live.clear()+ _release_index_worker(worker_key, worker_marker)+ raisedef _consume_terminal_exception(fut: "asyncio.Future[Any]") -> None:# The stalled thread may finish after we raise; retrieve its exception@@ -103,19 +147,17 @@ async def _run_with_stall_guard(done, _ = await asyncio.wait({future}, timeout=_INDEX_STALL_POLL_SECONDS)if done:return future.result()- # Reclaim the shared callback slot in case a concurrent job took it.- set_progress_callback(_heartbeat)stalled_for = time.monotonic() - last_progress["at"]if stalled_for > stall_timeout:- future.add_done_callback(_consume_terminal_exception)raise IndexingStallError(f"Indexing made no progress for {stalled_for:.0f}s while "- "embedding documents. The embedding provider may be accepting "- "requests without completing them; check the embedding "- "endpoint and retry."+ "embedding documents. Check the embedding endpoint; wait "+ "for the worker to finish or restart DeepTutor before retrying.")finally:- active = False+ progress_live.clear()+ if not future.done():+ future.add_done_callback(_consume_terminal_exception)class LlamaIndexPipeline:@@ -157,13 +199,14 @@ class LlamaIndexPipeline:async def initialize(self, kb_name: str, file_paths: List[str], **kwargs) -> bool:progress_callback = kwargs.get("progress_callback")image_progress_callback = kwargs.get("image_progress_callback")+ kb_dir = resolve_kb_dir(self.kb_base_dir, kb_name)+ _ensure_index_worker_available(str(kb_dir.resolve()))self._configure_settings()self.logger.info(f"Initializing KB '{kb_name}' with {len(file_paths)} files using LlamaIndex")- kb_dir = resolve_kb_dir(self.kb_base_dir, kb_name)signature = self._current_signature()storage_dir = resolve_storage_dir_for_rebuild(kb_dir, signature)@@ -186,6 +229,7 @@ class LlamaIndexPipeline:documents, storage_dir, show_progress=should_show_progress()),progress_callback=progress_callback,+ worker_key=str(kb_dir.resolve()),)self.logger.info(f"Index persisted to {storage_dir}")@@ -198,7 +242,9 @@ class LlamaIndexPipeline:except Exception as exc:self.logger.error(f"Failed to initialize KB: {exc}")self.logger.error(traceback.format_exc())- self._cleanup_failed_version_dir(storage_dir, signature)+ # A timed-out executor can still be writing to this directory.+ if not isinstance(exc, IndexingStallError):+ self._cleanup_failed_version_dir(storage_dir, signature)raisefinally:set_progress_callback(None)@@ -309,11 +355,12 @@ class LlamaIndexPipeline:async def add_documents(self, kb_name: str, file_paths: List[str], **kwargs) -> bool:progress_callback = kwargs.get("progress_callback")image_progress_callback = kwargs.get("image_progress_callback")+ kb_dir = resolve_kb_dir(self.kb_base_dir, kb_name)+ _ensure_index_worker_available(str(kb_dir.resolve()))self._configure_settings()self.logger.info(f"Adding {len(file_paths)} documents to KB '{kb_name}' using LlamaIndex")- kb_dir = resolve_kb_dir(self.kb_base_dir, kb_name)signature = self._current_signature()plan = storage.resolve_add_storage_plan(kb_dir, signature)@@ -334,6 +381,7 @@ class LlamaIndexPipeline:plan.existing_storage, plan.storage_dir, documents),progress_callback=progress_callback,+ worker_key=str(kb_dir.resolve()),)self.logger.info(f"Added {num_added} documents to existing index")if signature is not None and plan.storage_dir != plan.existing_storage:@@ -346,6 +394,7 @@ class LlamaIndexPipeline:documents, plan.storage_dir, show_progress=should_show_progress()),progress_callback=progress_callback,+ worker_key=str(kb_dir.resolve()),)self.logger.info(f"Created new index with {num_added} documents")if signature is not None:@@ -357,7 +406,9 @@ class LlamaIndexPipeline:except Exception as exc:self.logger.error(f"Failed to add documents: {exc}")self.logger.error(traceback.format_exc())- if plan.existing_storage is None or plan.storage_dir != plan.existing_storage:+ if not isinstance(exc, IndexingStallError) and (+ plan.existing_storage is None or plan.storage_dir != plan.existing_storage+ ):self._cleanup_failed_version_dir(plan.storage_dir, signature)raisefinally:diff --git a/tests/services/rag/test_llamaindex_embedding_roles.py b/tests/services/rag/test_llamaindex_embedding_roles.pyindex 0c1320619..0a2ef1ce4 100644--- a/tests/services/rag/test_llamaindex_embedding_roles.py+++ b/tests/services/rag/test_llamaindex_embedding_roles.py@@ -2,6 +2,7 @@from __future__ import annotations+from contextvars import copy_contextfrom types import SimpleNamespace@@ -47,3 +48,65 @@ def test_custom_embedding_passes_query_and_document_roles(monkeypatch) -> None:(["document"], "search_document"),(["one", "two"], "search_document"),]+++def test_index_progress_is_cumulative_across_outer_embedding_groups(monkeypatch) -> None:+ from llama_index.core.schema import TextNode++ from deeptutor.services.rag.pipelines.llamaindex import (+ embedding_adapter as embedding_module,+ )++ class _FakeClient:+ config = SimpleNamespace(binding="openai", model="test", batch_size=5)++ async def embed(self, texts, progress_callback=None, *, input_type=None):+ assert input_type == "search_document"+ total = (len(texts) + 4) // 5+ for batch in range(1, total + 1):+ progress_callback(batch, total)+ return [[1.0] for _ in texts]++ client = _FakeClient()+ monkeypatch.setattr(embedding_module, "get_embedding_client", lambda config=None: client)+ embedding = embedding_module.CustomEmbedding(embed_batch_size=10)+ events: list[tuple[int, int]] = []+ embedding_module.set_progress_callback(lambda n, total: events.append((n, total)))+ try:+ nodes = [TextNode(text=f"chunk {n}") for n in range(23)]+ assert len(embedding(nodes)) == 23+ finally:+ embedding_module.set_progress_callback(None)++ # LlamaIndex makes outer calls of 10, 10, and 3 chunks. Provider batches+ # are 2, 2, and 1; their user-visible sequence must never restart at 1.+ assert events == [(1, 5), (2, 5), (3, 5), (4, 5), (5, 5)]+++def test_old_embedding_context_cannot_pick_up_next_jobs_callback(monkeypatch) -> None:+ from deeptutor.services.rag.pipelines.llamaindex import (+ embedding_adapter as embedding_module,+ )++ class _FakeClient:+ config = SimpleNamespace(binding="openai", model="test", batch_size=1)++ async def embed(self, texts, progress_callback=None, *, input_type=None):+ progress_callback(1, 1)+ return [[1.0] for _ in texts]++ monkeypatch.setattr(embedding_module, "get_embedding_client", lambda config=None: _FakeClient())+ embedding = embedding_module.CustomEmbedding()+ old: list[tuple[int, int]] = []+ new: list[tuple[int, int]] = []+ embedding_module.set_progress_callback(lambda n, total: old.append((n, total)))+ old_worker_context = copy_context()+ embedding_module.set_progress_callback(lambda n, total: new.append((n, total)))+ try:+ old_worker_context.run(embedding._get_text_embeddings, ["old worker next group"])+ embedding._get_text_embeddings(["new worker"])+ finally:+ embedding_module.set_progress_callback(None)++ assert old == [(1, 1)]+ assert new == [(1, 1)]diff --git a/tests/services/rag/test_llamaindex_pipeline_stall.py b/tests/services/rag/test_llamaindex_pipeline_stall.pyindex 7f0704ff9..48ee69ef4 100644--- a/tests/services/rag/test_llamaindex_pipeline_stall.py+++ b/tests/services/rag/test_llamaindex_pipeline_stall.py@@ -12,6 +12,7 @@ from __future__ import annotationsimport asynciofrom contextvars import ContextVar+import threadingimport timefrom types import SimpleNamespacefrom typing import Any@@ -101,29 +102,22 @@ async def test_stall_guard_returns_when_progress_keeps_flowing(@pytest.mark.asyncio-async def test_stall_guard_survives_a_concurrent_job_taking_the_callback_slot(+async def test_stall_guard_keeps_its_worker_callback_when_another_job_starts(monkeypatch: pytest.MonkeyPatch,) -> None:- """A second indexing job must not make this one look stalled.-- ``set_progress_callback`` writes to the process-global LlamaIndex- ``Settings`` embed model, which holds exactly one callback. A concurrent- job overwrites ours, so without re-arming on each poll tick we would stop- observing progress we are still making and kill a perfectly healthy job- with a false ``IndexingStallError``.- """+ """Each executor's context keeps the callback captured at job start."""pipeline_module, _, _ = _llamaindex_modules()monkeypatch.setattr(pipeline_module, "_INDEX_STALL_POLL_SECONDS", 0.05)slot: dict = {}monkeypatch.setattr(pipeline_module, "set_progress_callback", lambda cb: slot.update(cb=cb))def displaced_then_moving():- # A concurrent indexing job claims the single shared slot.+ worker_callback = slot["cb"]+ # A concurrent job may start, but this worker retains its own callback.slot["cb"] = lambda *args, **kwargs: Noneend = time.monotonic() + 0.6while time.monotonic() < end:- # Batches notify whoever currently owns the slot.- slot["cb"](1, 1)+ worker_callback(1, 1)time.sleep(0.05)return "done"@@ -135,6 +129,108 @@ async def test_stall_guard_survives_a_concurrent_job_taking_the_callback_slot()+async def test_timed_out_worker_blocks_same_kb_retry_and_late_progress(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ """A timeout is terminal for progress, but its sync thread still owns the KB."""+ pipeline_module, _, _ = _llamaindex_modules()+ monkeypatch.setattr(pipeline_module, "_INDEX_STALL_POLL_SECONDS", 0.01)+ callback: dict = {}+ monkeypatch.setattr(pipeline_module, "set_progress_callback", lambda cb: callback.update(cb=cb))+ started = threading.Event()+ release = threading.Event()+ finished = threading.Event()+ events: list[tuple[int, int]] = []++ def delayed_worker():+ started.set()+ release.wait(timeout=3)+ callback["cb"](1, 1)+ finished.set()++ try:+ with pytest.raises(pipeline_module.IndexingStallError, match="no progress"):+ await pipeline_module._run_with_stall_guard(+ delayed_worker,+ progress_callback=lambda n, total: events.append((n, total)),+ stall_timeout=0.05,+ worker_key="kb-1478",+ )+ assert started.is_set()+ with pytest.raises(pipeline_module.IndexingStallError, match="still running"):+ await pipeline_module._run_with_stall_guard(lambda: "retry", worker_key="kb-1478")+ finally:+ release.set()++ assert await asyncio.to_thread(finished.wait, 2)+ for _ in range(100):+ if "kb-1478" not in pipeline_module._INDEX_WORKERS:+ break+ await asyncio.sleep(0.01)+ assert "kb-1478" not in pipeline_module._INDEX_WORKERS+ assert events == []+ assert (+ await pipeline_module._run_with_stall_guard(lambda: "retry", worker_key="kb-1478")+ == "retry"+ )+++async def test_different_kb_workers_keep_independent_progress(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ """A second KB must not displace the first one's heartbeat (#1478)."""+ pipeline_module, _, _ = _llamaindex_modules()+ from deeptutor.services.rag.pipelines.llamaindex.embedding_adapter import (+ _task_progress_callback,+ set_progress_callback,+ )++ monkeypatch.setattr(pipeline_module, "_INDEX_STALL_POLL_SECONDS", 0.01)+ monkeypatch.setattr(pipeline_module, "set_progress_callback", set_progress_callback)+ barrier = threading.Barrier(2)+ events: dict[str, list[tuple[int, int]]] = {"a": [], "b": []}++ def worker(name: str) -> str:+ callback = _task_progress_callback.get()+ assert callback is not None+ barrier.wait(timeout=2)+ end = time.monotonic() + 0.2+ while time.monotonic() < end:+ callback(1, 1)+ time.sleep(0.01)+ return name++ async def index(name: str) -> str:+ return await pipeline_module._run_with_stall_guard(+ lambda: worker(name),+ progress_callback=lambda n, total: events[name].append((n, total)),+ stall_timeout=0.08,+ worker_key=f"kb-{name}",+ )++ assert await asyncio.gather(index("a"), index("b")) == ["a", "b"]+ assert events["a"] and events["b"]+++async def test_busy_reindex_rejects_before_creating_an_empty_version(tmp_path, monkeypatch) -> None:+ pipeline_module, _, _ = _llamaindex_modules()+ pipeline = _make_pipeline(tmp_path, monkeypatch)+ key = str((tmp_path / "kb").resolve())+ marker = threading.Event()+ pipeline_module._INDEX_WORKERS[key] = marker+ try:+ with pytest.raises(pipeline_module.IndexingStallError, match="still running"):+ await pipeline.initialize("kb", ["doc.pdf"])+ finally:+ marker.set()+ pipeline_module._INDEX_WORKERS.pop(key, None)++ assert not (tmp_path / "kb" / "version-1").exists()++@pytest.mark.asyncioasync def test_stall_guard_forwards_user_progress_callback(monkeypatch: pytest.MonkeyPatch,diff --git a/web/hooks/useKnowledgeProgress.ts b/web/hooks/useKnowledgeProgress.tsindex 6607e3953..61dbb014b 100644--- a/web/hooks/useKnowledgeProgress.ts+++ b/web/hooks/useKnowledgeProgress.ts@@ -313,11 +313,13 @@ export function useKnowledgeProgress(options?: UseKnowledgeProgressOptions) {setTasksByKb((prev) => {const current = prev[kbName];if (!current || current.taskId !== taskId) return prev;+ const logs = appendTaskLog(current.logs, payload.message);+ if (logs === current.logs) return prev;return {...prev,[kbName]: {...current,- logs: [...current.logs, payload.message!],+ logs,},};});diff --git a/web/tests/knowledge-create-progress.spec.tsx b/web/tests/knowledge-create-progress.spec.tsxindex 6ad23abd5..d9417fc49 100644--- a/web/tests/knowledge-create-progress.spec.tsx+++ b/web/tests/knowledge-create-progress.spec.tsx@@ -113,6 +113,28 @@ it("subscribes without a task ID and tolerates opening a removed target", () =>expect(() => socket.onopen?.()).not.toThrow();});+it("keeps one log line when process and structured progress interleave", () => {+ const { result } = renderHook(() => useKnowledgeProgress());+ act(() =>+ result.current.resumeTask("papers", {+ task_id: "kb_reindex_1478",+ stage: "processing_documents",+ message: "Embedding batches: 1/5",+ }),+ );+ const stream = Stream.instances[0];+ act(() => {+ stream.emit("process_log", { message: "Embedding batches: 1/5" });+ stream.emit("progress", {+ task_id: "kb_reindex_1478",+ stage: "processing_documents",+ message: "Embedding batches: 1/5",+ });+ stream.emit("process_log", { message: "Embedding batches: 1/5" });+ });+ expect(result.current.tasksByKb.papers.logs).toEqual(["Embedding batches: 1/5"]);+});+it("restores the log stream when opening a processing KB, without clearing logs on refresh", async () => {const kbId = "workspace:study:kb:papers";fixture.list.mockResolvedValue([