DeepTutor: Lifelong Personalized Tutoring. https://deeptutor.info/.
Python ai-agentsai-tutorclawbotcli-tooldeepresearchinteractive-learninglearningmulti-agent-systemsrag
fix(knowledge): make long-document indexing recoverable and publication verified (#1612)
Bingxi Zhao (Frank)e79e1b0ad2a9
diff --git a/deeptutor/api/routers/knowledge.py b/deeptutor/api/routers/knowledge.pyindex e8fd777a6..d9234b30b 100644--- a/deeptutor/api/routers/knowledge.py+++ b/deeptutor/api/routers/knowledge.py@@ -1023,6 +1023,13 @@ async def run_initialization_task(indexed_count = len(FileTypeRouter.collect_supported_files(initializer.raw_dir, recursive=True))+ from deeptutor.knowledge.indexing_run import load_run++ receipt = load_run(initializer.raw_dir.parent)+ if receipt and receipt.get("provider") == "llamaindex":+ indexed_count = sum(+ doc.get("status") == "completed" for doc in receipt["documents"].values()+ )initializer.progress_tracker.update(ProgressStage.COMPLETED,@@ -3994,6 +4001,7 @@ async def run_reindex_task(manager._save_config()success = await rag_service.initialize(+ task_id=task_id,kb_name=kb_name,file_paths=file_paths,progress_callback=_on_progress,@@ -4106,6 +4114,20 @@ async def reindex_knowledge_base(kb_entry = _load_kb_entry_or_404(manager, kb_name)_assert_not_connected_kb(kb_name, kb_entry)force_reindex = str(kb_entry.get("status") or "").lower() == "error"+ from deeptutor.knowledge.indexing_run import visible_run++ previous_run = visible_run(kb_base_dir / kb_name)+ if previous_run:+ if previous_run["state"] == "running":+ raise HTTPException(+ status_code=409, detail="An indexing worker still owns this knowledge base."+ )+ force_reindex = force_reindex or previous_run["state"] in {+ "partial",+ "failed",+ "cancelled",+ "interrupted",+ }kb_provider = _validate_registered_provider(kb_entry.get("rag_provider") or DEFAULT_PROVIDER)@@ -4301,6 +4323,59 @@ async def retry_knowledge_base(raise HTTPException(status_code=500, detail=format_exception_message(e))[email protected]("/knowledge-bases/{kb_name}/indexing-run")+async def get_indexing_run(kb_name: str):+ """Pure durable diagnostics; reading never recreates a task or retries work."""+ from deeptutor.knowledge.indexing_run import visible_run++ resource = resolve_kb(kb_name)+ return {"run": visible_run(resource.base_dir / resource.name)}++[email protected]("/knowledge-bases/{kb_name}/indexing-run/{task_id}/cancel")+async def cancel_indexing_run(kb_name: str, task_id: str):+ from deeptutor.knowledge.indexing_run import request_cancel++ _, name, base_dir = _writable_kb(kb_name)+ if not request_cancel(base_dir / name, task_id):+ raise HTTPException(status_code=409, detail="This indexing run is no longer active.")+ return {+ "cancel_requested": True,+ "message": "Cancellation takes effect at the next safe boundary; completed parser outputs remain reusable.",+ }++[email protected]("/knowledge-bases/{kb_name}/indexing-readiness")+async def get_indexing_readiness(kb_name: str):+ from deeptutor.services.embedding.config import get_embedding_config+ from deeptutor.services.parsing import get_parse_service+ from deeptutor.services.parsing.engines.factory import get_parser++ resource = resolve_kb(kb_name)+ engine = get_parse_service().active_engine()+ parser = get_parser(engine)+ config = parser.resolve_config()+ report = parser.is_ready(config)+ embedding = get_embedding_config()+ return {+ "parser": engine,+ "ready": report.ready,+ "reason": report.reason,+ "message": report.message,+ "formats": sorted(parser.supported_formats()),+ "mode": getattr(config, "mode", None),+ "device": getattr(config, "device", None),+ "embedding_model": embedding.model,+ "embedding_configured": bool(embedding.model),+ "source_count": len(+ FileTypeRouter.collect_supported_files(+ resource.base_dir / resource.name / "raw", recursive=True+ )+ ),+ "note": "Readiness checks configuration and local prerequisites; runtime downloads, resource limits and provider responses can still fail.",+ }++@router.get("/knowledge-bases/{kb_name}/progress")async def get_progress(kb_name: str):"""Get initialization progress for a knowledge base"""diff --git a/deeptutor/knowledge/indexing_run.py b/deeptutor/knowledge/indexing_run.pynew file mode 100644index 000000000..43ce47016--- /dev/null+++ b/deeptutor/knowledge/indexing_run.py@@ -0,0 +1,447 @@+"""Durable per-KB indexing state and ownership at reliable boundaries (#1612).++Parsing caches remain the reusable work store. This journal records what was+accepted, parsed, failed, published, or unknown after interruption; it is not a+queue or a second parsing cache.+"""++from __future__ import annotations++import asyncio+from contextvars import ContextVar+from hashlib import sha256+import json+import os+from pathlib import Path+import re+import sys+import threading+import time+from typing import Any+from uuid import uuid4++try:+ import psutil+except ImportError: # CLI-only installs need no process-monitor dependency.+ psutil = None++from deeptutor.services.file_io import atomic_write_json++_CURRENT: ContextVar[IndexingRun | None] = ContextVar("indexing_run", default=None)+_TERMINAL = {"completed", "partial", "failed", "cancelled", "interrupted"}+++class IndexingBusyError(RuntimeError):+ pass+++class IndexingCancelled(RuntimeError):+ pass+++def current_run() -> IndexingRun | None:+ return _CURRENT.get()+++def _redact(detail: str) -> str:+ text = re.sub(r"(?i)(bearer\s+)[^\s]+", r"\1[redacted]", str(detail))+ text = re.sub(+ r"(?i)((?:api[_-]?key|token|secret|password|signature|authorization)\s*[=:]\s*)[^\s&]+",+ r"\1[redacted]",+ text,+ )+ text = re.sub(+ r'(?i)(["\']?(?:api[_-]?key|token|secret|password|signature|authorization)["\']?\s*[:=]\s*)["\'][^"\']*["\']',+ r'\1"[redacted]"',+ text,+ )+ text = re.sub(r"(https?://)[^/\s@]+:[^/\s@]+@", r"\1[redacted]@", text)+ return text[-1600:]+++def failure_code(exc: BaseException) -> str:+ text = str(exc).casefold()+ if isinstance(exc, (TimeoutError,)) or "timeout" in text or "timed out" in text:+ return "timeout"+ if isinstance(exc, MemoryError) or "out of memory" in text or "resource exhausted" in text:+ return "resource_exhaustion"+ if (+ isinstance(exc, (FileNotFoundError, ModuleNotFoundError))+ or "not installed" in text+ or "cli_missing" in text+ ):+ return "dependency_missing"+ if "unsupported" in text or "doesn't support" in text:+ return "unsupported_input"+ if "download" in text:+ return "model_download_failed"+ if "embedding" in text or "vector" in text:+ return "embedding_failed"+ if "no content" in text or "no output" in text or "invalid" in text:+ return "invalid_output"+ return "runtime_failure"+++def load_run(kb_dir: Path) -> dict[str, Any] | None:+ path = Path(kb_dir) / ".indexing-run.json"+ try:+ value = json.loads(path.read_text())+ except FileNotFoundError:+ return None+ except (OSError, ValueError) as exc:+ raise OSError("Indexing journal is unreadable; original file preserved.") from exc+ if (+ not isinstance(value, dict)+ or value.get("schema") != 1+ or not isinstance(value.get("documents"), dict)+ ):+ raise OSError("Indexing journal is invalid; original file preserved.")+ try:+ cancelled = json.loads((Path(kb_dir) / ".indexing-cancel.json").read_text())+ except FileNotFoundError:+ cancelled = {}+ except (OSError, ValueError) as exc:+ raise OSError(+ "Indexing cancellation state is unreadable; original file preserved."+ ) from exc+ value["cancel_requested"] = bool(+ isinstance(cancelled, dict) and cancelled.get("task_id") == value.get("task_id")+ )+ return value+++def _lease_is_held(kb_dir: Path) -> bool:+ try:+ with (Path(kb_dir) / ".indexing.lock").open("r+b") as lease:+ if sys.platform == "win32":+ import msvcrt++ msvcrt.locking(lease.fileno(), msvcrt.LK_NBLCK, 1)+ msvcrt.locking(lease.fileno(), msvcrt.LK_UNLCK, 1)+ else:+ import fcntl++ fcntl.flock(lease.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB)+ fcntl.flock(lease.fileno(), fcntl.LOCK_UN)+ return False+ except FileNotFoundError:+ return False+ except OSError:+ return True+++def visible_run(kb_dir: Path) -> dict[str, Any] | None:+ """Pure projection: a crashed owner's in-flight work becomes unknown."""+ value = load_run(kb_dir)+ if value is None or value.get("state") in _TERMINAL:+ return value+ try:+ alive = (+ psutil.Process(int(value["pid"])).create_time() == value["process_started_at"]+ if psutil is not None+ else _lease_is_held(kb_dir)+ )+ except Exception:+ alive = False+ if alive:+ return value+ documents = {+ key: {+ **doc,+ "status": "unknown"+ if doc.get("status") in {"pending", "parsing", "embedding"}+ else doc.get("status"),+ }+ for key, doc in value["documents"].items()+ }+ return {+ **value,+ "state": "interrupted",+ "documents": documents,+ "recovery": "Retry reuses validated parser outputs and completed slices. The interrupted uncheckpointed stage restarts.",+ }+++class IndexingRun:+ def __init__(+ self,+ kb_dir: Path,+ sources: list[str],+ *,+ task_id: str = "",+ action: str = "rebuild",+ provider: str = "",+ reporter=None,+ ):+ self.kb_dir = Path(kb_dir)+ self.path = self.kb_dir / ".indexing-run.json"+ self.task_id = task_id or f"index_{uuid4().hex}"+ self.action = action+ self.provider = provider+ self.sources = sources+ self.reporter = reporter+ self._mutex = threading.RLock()+ self.active = False+ self.data: dict[str, Any] = {}+ self._lock = None+ self._token = None+ self._source_paths: dict[str, Path] = {}++ def __enter__(self):+ return self._start()++ async def __aenter__(self):+ future = asyncio.create_task(asyncio.to_thread(self._start, False))+ try:+ await asyncio.shield(future)+ except asyncio.CancelledError:+ await future+ self.finish("cancelled")+ self.active = False+ self._release()+ raise+ self._token = _CURRENT.set(self)+ return self++ async def __aexit__(self, kind, exc, traceback):+ self.__exit__(kind, exc, traceback)++ def _start(self, bind_context=True):+ self.kb_dir.mkdir(parents=True, exist_ok=True)+ self._lock = (self.kb_dir / ".indexing.lock").open("a+b")+ try:+ if sys.platform == "win32":+ import msvcrt++ self._lock.seek(0)+ if not self._lock.read(1):+ self._lock.write(b"\0")+ self._lock.flush()+ self._lock.seek(0)+ msvcrt.locking(self._lock.fileno(), msvcrt.LK_NBLCK, 1)+ else:+ import fcntl++ fcntl.flock(self._lock.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB)+ except OSError as exc:+ self._lock.close()+ raise IndexingBusyError(+ "An indexing worker still owns this knowledge base. Wait for it to settle before retrying."+ ) from exc+ try:+ previous = visible_run(self.kb_dir)+ now = time.time()+ self.data = {+ "schema": 1,+ "task_id": self.task_id,+ "action": self.action,+ "provider": self.provider,+ "pid": os.getpid(),+ "process_started_at": psutil.Process().create_time()+ if psutil is not None+ else None,+ "state": "running",+ "phase": "preflight",+ "started_at": now,+ "last_activity_at": now,+ "last_progress_at": now,+ "documents": {},+ "previous_task_id": previous.get("task_id") if previous else None,+ "recovery": "Validated parser caches and completed slices are reused. Uncheckpointed stages restart.",+ "cancel_requested": False,+ "usable_version": None,+ }+ from deeptutor.services.rag.file_routing import FileTypeRouter++ candidates = list(dict.fromkeys(self.sources))+ raw = self.kb_dir / "raw"+ if self.action != "upload" and raw.is_dir():+ candidates.extend(+ str(path)+ for path in raw.rglob("*")+ if path.is_file() and not path.is_symlink() and str(path) not in candidates+ )+ classification = FileTypeRouter.classify_files(candidates)+ unsupported = set(classification.unsupported)+ for name in candidates:+ source = Path(name)+ key = self.source_key(source)+ self._source_paths[key] = source+ try:+ digest = self._hash(source)+ reason = "unsupported_input" if name in unsupported else ""+ status = "excluded" if reason else "pending"+ except OSError as exc:+ digest, reason, status = "", _redact(str(exc)), "failed"+ self.data["documents"][key] = {+ "status": status,+ "reason": reason,+ "source_hash": digest,+ "parse_reused": False,+ }+ self.active = True+ if bind_context:+ self._token = _CURRENT.set(self)+ self._save()+ return self+ except BaseException:+ self.active = False+ if self._token is not None:+ _CURRENT.reset(self._token)+ self._release()+ raise++ def __exit__(self, kind, exc, traceback):+ try:+ if exc is not None:+ self.finish(+ "partial"+ if self.data.get("usable_version")+ else "cancelled"+ if isinstance(exc, IndexingCancelled)+ or (load_run(self.kb_dir) or {}).get("cancel_requested")+ else "failed",+ version=self.data.get("usable_version"),+ error=exc,+ )+ elif self.data.get("state") not in _TERMINAL:+ self.finish("failed", error=RuntimeError("Index readiness was not verified."))+ finally:+ self.active = False+ if self._token is not None:+ _CURRENT.reset(self._token)+ self._release()++ def _release(self):+ if self._lock is not None and not self._lock.closed:+ if sys.platform == "win32":+ import msvcrt++ self._lock.seek(0)+ msvcrt.locking(self._lock.fileno(), msvcrt.LK_UNLCK, 1)+ else:+ import fcntl++ fcntl.flock(self._lock.fileno(), fcntl.LOCK_UN)+ self._lock.close()++ def source_key(self, source: Path) -> str:+ try:+ return source.resolve().relative_to((self.kb_dir / "raw").resolve()).as_posix()+ except ValueError:+ return (+ f"external-{sha256(str(source.resolve()).encode()).hexdigest()[:12]}/{source.name}"+ )++ @staticmethod+ def _hash(source: Path) -> str:+ digest = sha256()+ with source.open("rb") as stream:+ for chunk in iter(lambda: stream.read(1024 * 1024), b""):+ digest.update(chunk)+ return digest.hexdigest()++ def check(self):+ if not self.active:+ raise IndexingCancelled("This indexing worker no longer owns the run.")+ stored = load_run(self.kb_dir)+ if stored and stored.get("task_id") != self.task_id:+ raise IndexingCancelled("A newer run owns the indexing journal.")+ if stored and stored.get("cancel_requested"):+ self.data["cancel_requested"] = True+ raise IndexingCancelled(+ "Cancellation reached a safe boundary; validated completed parsing remains reusable."+ )++ def phase(+ self,+ phase: str,+ *,+ source: Path | None = None,+ current: int | None = None,+ total: int | None = None,+ activity: str = "",+ ):+ with self._mutex:+ self.check()+ now = time.time()+ changed = self.data.get("phase") != phase or (+ current is not None and current != self.data.get("phase_current")+ )+ self.data.update(+ {+ "phase": phase,+ "last_activity_at": now,+ "current_document": self.source_key(source)+ if source+ else self.data.get("current_document"),+ "phase_current": current,+ "phase_total": total,+ "activity": _redact(activity),+ }+ )+ if changed:+ self.data["last_progress_at"] = now+ self._save()++ def document(self, source: Path, status: str, **details):+ with self._mutex:+ if not self.active:+ return+ stored = load_run(self.kb_dir)+ if stored and stored.get("task_id") != self.task_id:+ return+ key = self.source_key(source)+ doc = self.data["documents"].setdefault(key, {})+ doc.update({"status": status, "updated_at": time.time(), **details})+ self.data["last_progress_at"] = time.time()+ self._save()++ def verify_sources(self):+ self.check()+ for key, doc in self.data["documents"].items():+ if doc.get("status") in {"parsed", "embedding", "completed"}:+ if self._hash(self._source_paths[key]) != doc.get("source_hash"):+ raise RuntimeError(+ f"Source changed during indexing: {key}. Retry with the current source."+ )++ def finish(self, state: str, *, version: str | None = None, error: BaseException | None = None):+ with self._mutex:+ if not self.active:+ return+ stored = load_run(self.kb_dir)+ if stored and stored.get("task_id") != self.task_id:+ return+ for doc in self.data["documents"].values():+ if doc.get("status") in {"pending", "parsing", "embedding", "parsed"}:+ doc["status"] = "unknown"+ self.data.update(+ {+ "state": state,+ "finished_at": time.time(),+ "usable_version": version,+ "error_code": failure_code(error) if error else None,+ "error": _redact(str(error)) if error else None,+ }+ )+ self._save()++ def _save(self):+ atomic_write_json(self.path, self.data)+ if self.reporter is not None:+ try:+ self.reporter(dict(self.data))+ except Exception:+ # Notification delivery cannot invalidate the durable checkpoint.+ pass+++def request_cancel(kb_dir: Path, task_id: str) -> bool:+ value = load_run(kb_dir)+ if value is None or value.get("task_id") != task_id or value.get("state") in _TERMINAL:+ return False+ atomic_write_json(+ Path(kb_dir) / ".indexing-cancel.json", {"task_id": task_id, "requested_at": time.time()}+ )+ return Truediff --git a/deeptutor/knowledge/initializer.py b/deeptutor/knowledge/initializer.pyindex 74ee212f3..364bd2ba3 100644--- a/deeptutor/knowledge/initializer.py+++ b/deeptutor/knowledge/initializer.py@@ -107,6 +107,15 @@ class KnowledgeBaseInitializer:metadata["last_indexed_count"] = len(FileTypeRouter.collect_supported_files(self.raw_dir, recursive=True))+ if provider == "llamaindex":+ from deeptutor.knowledge.indexing_run import load_run++ receipt = load_run(self.kb_dir)+ if receipt:+ metadata["last_indexed_count"] = sum(+ doc.get("status") == "completed" for doc in receipt["documents"].values()+ )+ metadata["indexing_state"] = receipt["state"]metadata["last_indexed_action"] = "create"atomic_write_json(metadata_file, metadata)@@ -238,6 +247,7 @@ class KnowledgeBaseInitializer:success = await rag_service.initialize(kb_name=self.kb_name,file_paths=file_paths,+ task_id=self.progress_tracker.task_id,progress_callback=_on_progress,image_progress_callback=_on_image_progress,indexing_snapshot=self.indexing_snapshot,diff --git a/deeptutor/services/parsing/engines/mineru/backend.py b/deeptutor/services/parsing/engines/mineru/backend.pyindex b0a0035c4..4f81dfb93 100644--- a/deeptutor/services/parsing/engines/mineru/backend.py+++ b/deeptutor/services/parsing/engines/mineru/backend.py@@ -35,6 +35,8 @@ _LOCAL_CLI_COMMANDS = ("mineru", "magic-pdf")#: hint is deliberately limited to ``CLI_MISSING``: a runtime failure must not#: be reported as a missing installation (issue #1612, outcome 4)._LOCAL_FAILURE_MESSAGES: dict[LocalParseReason, str] = {+ LocalParseReason.TIMEOUT: "Local MinerU parsing timed out: {detail}",+ LocalParseReason.CANCELLED: "Local MinerU parsing was cancelled: {detail}",LocalParseReason.CLI_MISSING: ("Local MinerU parsing failed. Ensure MinerU is installed ""(`pip install -U 'mineru[all]>=3.4.5'`) or switch to cloud mode in "diff --git a/deeptutor/services/parsing/engines/mineru/local.py b/deeptutor/services/parsing/engines/mineru/local.pyindex 236d3fa7f..f3861b535 100644--- a/deeptutor/services/parsing/engines/mineru/local.py+++ b/deeptutor/services/parsing/engines/mineru/local.py@@ -12,6 +12,7 @@ import shutilimport subprocessimport sysimport tempfile+import threadingimport timefrom uuid import uuid4@@ -24,6 +25,8 @@ from .formats import MINERU_SUPPORTED_FORMATS# progress that universal-newline decoding turns into many lines per second;# without a floor the trace panel gets flooded during model downloads._ON_OUTPUT_MIN_INTERVAL = 0.5+_LOCAL_PARSE_IDLE_TIMEOUT_SECONDS = 600+_LOCAL_PARSE_TIMEOUT_SECONDS = 7200#: Upper bound on the failure excerpt carried back to callers: long enough for#: a useful stderr tail, short enough to fit inside an error message.@@ -49,6 +52,8 @@ class LocalParseReason(StrEnum):INPUT_MISSING = "input_missing"UNSUPPORTED_INPUT = "unsupported_input"LEGACY_CLI_INPUT = "legacy_cli_input"+ TIMEOUT = "timeout"+ CANCELLED = "cancelled"NONZERO_EXIT = "nonzero_exit"NO_ARTIFACTS = "no_artifacts"EXCEPTION = "exception"@@ -204,6 +209,9 @@ def parse_document_with_mineru_result(atomic_write_json(state_path, {"source": source_file.name, "state": "running"})process = Noneprocess_finished = False+ watchdog_stop = threading.Event()+ watchdog = None+ interrupted: list[LocalParseReason] = []result = LocalParseResult.failure(LocalParseReason.EXCEPTION, "parse interrupted")try:cmd = [mineru_cmd, "-p", str(source_file), "-o", str(temp_output)]@@ -225,11 +233,44 @@ def parse_document_with_mineru_result()tail: deque[str] = deque(maxlen=40)last_emit = 0.0+ from deeptutor.knowledge.indexing_run import IndexingCancelled, current_run++ run = current_run()+ activity = [time.monotonic()]+ if run is not None:+ started = activity[0]++ def watch():+ while not watchdog_stop.wait(0.25):+ try:+ run.check()+ except IndexingCancelled:+ interrupted.append(LocalParseReason.CANCELLED)+ except OSError:+ interrupted.append(LocalParseReason.EXCEPTION)+ if (+ time.monotonic() - activity[0] > _LOCAL_PARSE_IDLE_TIMEOUT_SECONDS+ or time.monotonic() - started > _LOCAL_PARSE_TIMEOUT_SECONDS+ ):+ interrupted.append(LocalParseReason.TIMEOUT)+ if interrupted:+ try:+ process.terminate()+ process.wait(timeout=3)+ except subprocess.TimeoutExpired:+ process.kill()+ except OSError:+ pass+ return++ watchdog = threading.Thread(target=watch, daemon=True)+ watchdog.start()assert process.stdout is not Nonefor raw_line in process.stdout:line = raw_line.strip()if not line:continue+ activity[0] = time.monotonic()tail.append(line)if on_output is not None:now = time.monotonic()@@ -244,6 +285,12 @@ def parse_document_with_mineru_result(returncode = process.wait()process_finished = True+ if interrupted:+ result = LocalParseResult.failure(+ interrupted[0],+ "Cancellation or parse time limit reached; completed checkpoints remain reusable.",+ )+ return resultif returncode != 0:print("✗ MinerU parsing failed:")print("\n".join(tail))@@ -309,6 +356,9 @@ def parse_document_with_mineru_result()return resultfinally:+ watchdog_stop.set()+ if watchdog is not None:+ watchdog.join(timeout=4)# Stop an interrupted child before another attempt can publish output.if process is not None and not process_finished:try:diff --git a/deeptutor/services/parsing/service.py b/deeptutor/services/parsing/service.pyindex 2db211b24..5834ea086 100644--- a/deeptutor/services/parsing/service.py+++ b/deeptutor/services/parsing/service.py@@ -101,7 +101,35 @@ class ParseService:supported = get_parser(engine_name).supported_formats()return not supported or _matches_supported_format(source_path, supported)- def parse(+ def parse(self, source_path: str | Path, *, engine=None, on_output=None) -> ParsedDocument:+ from deeptutor.knowledge.indexing_run import _redact, current_run, failure_code++ run = current_run()+ source = Path(source_path)+ if run:+ run.phase("parsing", source=source)+ run.document(source, "parsing")++ def output(line):+ if run:+ run.phase("parsing", source=source, activity=line)+ if on_output:+ on_output(line)++ try:+ result = self._parse(source, engine=engine, on_output=output)+ if run:+ run.check()+ run.document(+ source, "parsed", parser=result.engine, parser_signature=result.parser_signature+ )+ return result+ except Exception as exc:+ if run:+ run.document(source, "failed", reason=failure_code(exc), detail=_redact(str(exc)))+ raise++ def _parse(self,source_path: str | Path,*,@@ -161,6 +189,10 @@ class ParseService:markdown, blocks, asset_dir = "", None, Noneif markdown.strip() or blocks:logger.info("Parse cache hit for %s (%s/%s)", source_path.name, engine_name, sig)+ from deeptutor.knowledge.indexing_run import current_run++ if run := current_run():+ run.document(source_path, "parsed", parse_reused=True)return ParsedDocument(markdown=markdown,blocks=blocks,diff --git a/deeptutor/services/rag/index_probe.py b/deeptutor/services/rag/index_probe.pyindex c55458e4b..a2a5704ce 100644--- a/deeptutor/services/rag/index_probe.py+++ b/deeptutor/services/rag/index_probe.py@@ -143,6 +143,15 @@ def _inspect_llamaindex(storage_dir: Path) -> ProviderIndexProbe:"LlamaIndex storage directory does not exist.",)+ if (storage_dir / ".building.json").exists():+ meta = _read_json(storage_dir / "meta.json")+ if not meta or meta.get("state") != "published" or not meta.get("readiness_verified"):+ return ProviderIndexProbe(+ DEFAULT_PROVIDER,+ str(storage_dir),+ False,+ "Candidate index has not passed persistence verification.",+ )docstore = storage_dir / "docstore.json"index_store = storage_dir / "index_store.json"vector_stores = sorted(path.name for path in storage_dir.glob("*vector_store.json"))diff --git a/deeptutor/services/rag/index_versioning.py b/deeptutor/services/rag/index_versioning.pyindex 0e50ba367..644e314d2 100644--- a/deeptutor/services/rag/index_versioning.py+++ b/deeptutor/services/rag/index_versioning.py@@ -153,6 +153,8 @@ def _entry_from_flat_version(version_dir: Path) -> dict[str, Any]:or (version_dir / "deeptutor_ingress").exists())ready = has_provider_output and not is_unpublished_lightrag_candidate+ if (version_dir / ".building.json").exists():+ ready = ready and bool(stored_meta and stored_meta.get("state") == "published")if stored_meta is not None and stored_meta.get("provider") == "lightrag":adapter_schema = stored_meta.get("lightrag_adapter_schema")if adapter_schema is not None:@@ -317,8 +319,11 @@ def write_version_meta("layout": "flat" if target.parent == kb_dir else "nested_legacy","created_at": datetime.now(timezone.utc).replace(tzinfo=None).isoformat() + "Z",}- with open(target / META_FILENAME, "w", encoding="utf-8") as handle:- json.dump(payload, handle, indent=2, ensure_ascii=False)+ from deeptutor.services.file_io import atomic_write_json++ if (target / ".building.json").exists():+ payload.update(provider="llamaindex", state="published", readiness_verified=True)+ atomic_write_json(target / META_FILENAME, payload)def resolve_storage_dir_for_read(@@ -356,7 +361,9 @@ def resolve_storage_dir_for_write(kb_dir: Path, signature: Optional[EmbeddingSigreturn target-def resolve_storage_dir_for_rebuild(kb_dir: Path, signature: Optional[EmbeddingSignature]) -> Path:+def resolve_storage_dir_for_rebuild(+ kb_dir: Path, signature: Optional[EmbeddingSignature], *, publication_guard: bool = False+) -> Path:"""Return a fresh flat storage dir for a full index rebuild.Incremental writes reuse a matching flat version, but a full rebuild should@@ -365,6 +372,16 @@ def resolve_storage_dir_for_rebuild(kb_dir: Path, signature: Optional[EmbeddingSrebuild remain diagnosable and avoids stale vector-store files."""_ = signature- target = _next_flat_version_dir(kb_dir)- target.mkdir(parents=True, exist_ok=True)+ kb_dir.mkdir(parents=True, exist_ok=True)+ while True:+ target = _next_flat_version_dir(kb_dir)+ try:+ target.mkdir()+ break+ except FileExistsError:+ continue+ from deeptutor.services.file_io import atomic_write_json++ if publication_guard:+ atomic_write_json(target / ".building.json", {"state": "building"})return targetdiff --git a/deeptutor/services/rag/pipelines/llamaindex/document_loader.py b/deeptutor/services/rag/pipelines/llamaindex/document_loader.pyindex 26f4d5625..103488727 100644--- a/deeptutor/services/rag/pipelines/llamaindex/document_loader.py+++ b/deeptutor/services/rag/pipelines/llamaindex/document_loader.py@@ -172,6 +172,30 @@ class LlamaIndexDocumentLoader:for file_path_str in classification.unsupported:self.logger.warning(f"Skipped unsupported file: {Path(file_path_str).name}")+ from deeptutor.knowledge.indexing_run import current_run++ if run := current_run():+ accepted = {+ str(Path(doc.metadata["file_path"]).resolve())+ for doc in documents+ if isinstance(getattr(doc, "metadata", None), dict)+ and doc.metadata.get("file_path")+ }+ for name in (+ classification.parser_files + classification.text_files + classification.image_files+ ):+ source = Path(name)+ if str(source.resolve()) in accepted:+ run.document(source, "embedding")+ elif (+ run.data["documents"].get(run.source_key(source), {}).get("status") != "failed"+ ):+ run.document(+ source,+ "failed",+ reason="invalid_output",+ detail="No usable document content.",+ )return documentsdef _parse_document(diff --git a/deeptutor/services/rag/pipelines/llamaindex/pipeline.py b/deeptutor/services/rag/pipelines/llamaindex/pipeline.pyindex fe280e9e8..798a1b466 100644--- a/deeptutor/services/rag/pipelines/llamaindex/pipeline.py+++ b/deeptutor/services/rag/pipelines/llamaindex/pipeline.py@@ -209,7 +209,7 @@ class LlamaIndexPipeline:)signature = self._current_signature()- storage_dir = resolve_storage_dir_for_rebuild(kb_dir, signature)+ storage_dir = resolve_storage_dir_for_rebuild(kb_dir, signature, publication_guard=True)try:await self._verify_embedding_connectivity()@@ -242,8 +242,7 @@ class LlamaIndexPipeline:)self.logger.info(f"Index persisted to {storage_dir}")- if signature is not None:- write_version_meta(kb_dir, signature, storage_dir=storage_dir)+ await self._publish_verified(kb_dir, storage_dir, signature, documents)indexed_file_callback = kwargs.get("indexed_file_callback")if indexed_file_callback is not None:@@ -437,8 +436,7 @@ class LlamaIndexPipeline:worker_key=str(kb_dir.resolve()),)self.logger.info(f"Added {num_added} documents to existing index")- if signature is not None and plan.storage_dir != plan.existing_storage:- write_version_meta(kb_dir, signature, storage_dir=plan.storage_dir)+else:self.logger.info(f"Creating new index with {len(documents)} documents...")plan.storage_dir.mkdir(parents=True, exist_ok=True)@@ -450,9 +448,8 @@ class LlamaIndexPipeline:worker_key=str(kb_dir.resolve()),)self.logger.info(f"Created new index with {num_added} documents")- if signature is not None:- write_version_meta(kb_dir, signature, storage_dir=plan.storage_dir)+ await self._publish_verified(kb_dir, plan.storage_dir, signature, documents)self.logger.info(f"Successfully added documents to KB '{kb_name}'")return True@@ -467,6 +464,35 @@ class LlamaIndexPipeline:finally:set_progress_callback(None)+ async def _publish_verified(self, kb_dir, storage_dir, signature, documents):+ from deeptutor.knowledge.indexing_run import current_run+ from deeptutor.services.file_io import atomic_write_json++ run = current_run()+ if run:+ run.phase("verifying")+ await asyncio.to_thread(storage.verify_persisted_index, storage_dir)+ if run:+ await asyncio.to_thread(run.verify_sources)+ if signature is not None:+ write_version_meta(kb_dir, signature, storage_dir=storage_dir)+ else:+ atomic_write_json(+ storage_dir / "meta.json",+ {+ "version": storage_dir.name,+ "provider": "llamaindex",+ "state": "published",+ "readiness_verified": True,+ },+ )+ if run:+ run.data["usable_version"] = storage_dir.name+ for document in documents:+ path = (getattr(document, "metadata", None) or {}).get("file_path")+ if path:+ run.document(Path(path), "completed")+async def delete(self, kb_name: str) -> bool:kb_dir = resolve_kb_dir(self.kb_base_dir, kb_name)deleted = storage.delete_kb_dir(kb_dir)diff --git a/deeptutor/services/rag/pipelines/llamaindex/storage.py b/deeptutor/services/rag/pipelines/llamaindex/storage.pyindex 4dfb406ea..499176304 100644--- a/deeptutor/services/rag/pipelines/llamaindex/storage.py+++ b/deeptutor/services/rag/pipelines/llamaindex/storage.py@@ -15,9 +15,8 @@ from typing import Anyfrom deeptutor.services.embedding.validation import validate_embedding_batchfrom deeptutor.services.rag.index_versioning import (EmbeddingSignature,- find_matching_version,resolve_storage_dir_for_read,- resolve_storage_dir_for_write,+ resolve_storage_dir_for_rebuild,)from . import ingestion, retrievers, vector_store@@ -59,32 +58,12 @@ def cleanup_failed_version_dir(storage_dir: Path) -> bool:def resolve_add_storage_plan(kb_dir: Path, signature: EmbeddingSignature | None) -> AddStoragePlan:"""Choose existing/new storage dirs for incremental adds."""- matching_version = find_matching_version(kb_dir, signature) if signature is not None else None- existing_storage = (- _storage_path_from_version_entry(matching_version) if matching_version else None+ existing = resolve_storage_dir_for_read(kb_dir, signature)+ return AddStoragePlan(+ existing_storage=existing,+ storage_dir=resolve_storage_dir_for_rebuild(kb_dir, signature, publication_guard=True),)- if matching_version and existing_storage and matching_version.get("layout") == "flat":- return AddStoragePlan(existing_storage=existing_storage, storage_dir=existing_storage)-- if matching_version and existing_storage:- return AddStoragePlan(- existing_storage=existing_storage,- storage_dir=resolve_storage_dir_for_write(kb_dir, signature),- )-- fallback_storage = resolve_storage_dir_for_read(kb_dir, signature)- existing_storage = fallback_storage- fallback_is_flat = (- fallback_storage is not None- and fallback_storage.parent == kb_dir- and fallback_storage.name.startswith("version-")- )- storage_dir = (- fallback_storage if fallback_is_flat else resolve_storage_dir_for_write(kb_dir, signature)- )- return AddStoragePlan(existing_storage=existing_storage, storage_dir=storage_dir)-def create_index(documents: list[Any], storage_dir: Path, *, show_progress: bool | None = None@@ -316,3 +295,26 @@ def delete_kb_dir(kb_dir: Path) -> bool:shutil.rmtree(kb_dir)return Truereturn False+++def verify_persisted_index(storage_dir: Path) -> None:+ index = vector_store.load_index(storage_dir)+ _validate_persisted_embeddings(index, storage_dir)+ if not index.docstore.docs:+ raise ValueError("Persisted index contains no retrievable nodes.")+ if not getattr(index, "index_struct", None):+ raise ValueError("Persisted index has no index structure.")+ vectors = [values for _, values in _iter_index_embedding_dicts(index) if values]+ if vectors:+ sample = next(iter(vectors[0].values()))+ else:+ client = getattr(index.vector_store, "client", None)+ if client is None or not getattr(client, "ntotal", 0):+ raise ValueError("Persisted index contains no embedding vectors.")+ sample = client.reconstruct(0).tolist()+ from llama_index.core.schema import QueryBundle++ if not index.as_retriever(similarity_top_k=1).retrieve(+ QueryBundle(query_str="", embedding=sample)+ ):+ raise ValueError("Persisted index failed a retrieval probe with a stored vector.")diff --git a/deeptutor/services/rag/service.py b/deeptutor/services/rag/service.pyindex 5fb4a3050..49da9488d 100644--- a/deeptutor/services/rag/service.py+++ b/deeptutor/services/rag/service.py@@ -66,21 +66,62 @@ class RAGService:@with_kb_embeddingasync def initialize(self, kb_name: str, file_paths: List[str], **kwargs) -> bool:- provider = self._resolve_provider(kb_name)- self.logger.info(f"Initializing KB '{kb_name}' (provider={provider})")- pipeline = self._get_pipeline(provider)- return await pipeline.initialize(kb_name=kb_name, file_paths=file_paths, **kwargs)+ return await self._index(kb_name, file_paths, "initialize", kwargs)@with_kb_embeddingasync def add_documents(self, kb_name: str, file_paths: List[str], **kwargs) -> bool:+ return await self._index(kb_name, file_paths, "add_documents", kwargs)++ async def _index(self, kb_name, file_paths, method, kwargs):+ from deeptutor.knowledge.indexing_run import IndexingRun, current_run++ from .index_probe import latest_ready_provider_version+ from .kb_paths import resolve_kb_dir+provider = self._resolve_provider(kb_name)- self.logger.info(- f"Adding {len(file_paths)} document(s) to KB '{kb_name}' (provider={provider})"- )pipeline = self._get_pipeline(provider)- if not hasattr(pipeline, "add_documents"):- return await pipeline.initialize(kb_name=kb_name, file_paths=file_paths, **kwargs)- return await pipeline.add_documents(kb_name=kb_name, file_paths=file_paths, **kwargs)+ operation = getattr(pipeline, method, None) or pipeline.initialize+ if current_run() is not None:+ return await operation(kb_name=kb_name, file_paths=file_paths, **kwargs)+ task_id = kwargs.pop("task_id", "")+ async with IndexingRun(+ resolve_kb_dir(self.kb_base_dir, kb_name),+ file_paths,+ task_id=task_id,+ action="upload" if method == "add_documents" else "rebuild",+ provider=provider,+ ) as run:+ receipt_callback = kwargs.get("indexed_file_callback")++ def indexed(paths):+ for path in paths:+ run.document(Path(path), "completed")+ if receipt_callback:+ receipt_callback(paths)++ kwargs["indexed_file_callback"] = indexed+ callback = kwargs.get("progress_callback")++ def progress(current, total):+ run.phase("embedding", current=current, total=total)+ if callback:+ callback(current, total)++ kwargs["progress_callback"] = progress+ success = await operation(kb_name=kb_name, file_paths=file_paths, **kwargs)+ if not run.data.get("usable_version"):+ run.check()+ if not success:+ return False+ version = latest_ready_provider_version(run.kb_dir, provider)+ if version is None:+ raise RuntimeError("Index output did not pass the provider readiness check.")+ partial = any(+ doc.get("status") in {"failed", "unknown", "pending", "parsing"}+ for doc in run.data["documents"].values()+ )+ run.finish("partial" if partial else "completed", version=version.get("version"))+ return True@with_kb_embeddingasync def search(diff --git a/docs/INDEXING_RECOVERY.md b/docs/INDEXING_RECOVERY.mdnew file mode 100644index 000000000..7e3e71117--- /dev/null+++ b/docs/INDEXING_RECOVERY.md@@ -0,0 +1,37 @@+# Long-document indexing and recovery++Each knowledge base records its latest indexing run in `.indexing-run.json`.+The Files/detail view exposes **Indexing recovery details**, including individual+source outcomes, current phase and phase-local counts, elapsed runtime, time+since output/activity, and time since measurable progress. A phase reaching its+last batch is not proof that the whole index is ready. Unknown work remains+unknown after interruption; it is never counted as a successful document.++`GET /api/knowledge-bases/{id}/indexing-readiness` performs a lightweight parser,+format and embedding-configuration check without downloading models or sending+a provider request. Readiness does not guarantee later downloads, available RAM+or GPU memory, nor a provider's runtime response. Runtime failures retain their+bounded, credential-redacted reasons in the durable run.++Retry uses the existing content-addressed parser cache: source bytes and parser+signature must match, and usable text or blocks must still exist. MinerU cloud+PDF slices reuse only validated checkpoints. Changing embeddings can reuse+parsing, but builds a separate index version for the new embedding signature.+Interrupted stages without a usable checkpoint restart; the journal itself is+not a second cache and does not fabricate partial embeddings.++Cancellation is task-specific and takes effect at a safe boundary. Managed+local MinerU processes also observe cancellation and fail after 600 seconds+without output or 7200 seconds total. Other synchronous engines cannot be+interrupted arbitrarily; cancellation remains pending until they return. A+stalled LlamaIndex executor is barred from overlapping same-process retries+until it actually exits, and cannot publish its candidate after losing ownership.++LlamaIndex rebuilds and incremental additions use fresh version directories.+A candidate stays unpublished until its persisted index reloads, its vectors+validate and a retrieval probe using a stored vector returns a node. Publication+metadata is written atomically. Source changes during indexing reject publication.+A failed or partial candidate does not overwrite a previous queryable version.+Completed and failed sources are reported separately, including partial success.+The durable journal is scoped to the active user/workspace knowledge base;+linked/read-only knowledge bases cannot be cancelled through mutation routes.diff --git a/tests/knowledge/test_indexing_run.py b/tests/knowledge/test_indexing_run.pynew file mode 100644index 000000000..0ecfe6556--- /dev/null+++ b/tests/knowledge/test_indexing_run.py@@ -0,0 +1,179 @@+"""Durable outcomes, interruption, cancellation and publication (#1612)."""++import json+from pathlib import Path+from types import SimpleNamespace++import pytest++from deeptutor.knowledge.indexing_run import (+ IndexingBusyError,+ IndexingCancelled,+ IndexingRun,+ current_run,+ load_run,+ request_cancel,+ visible_run,+)+from deeptutor.services.rag.index_versioning import (+ EmbeddingSignature,+ list_kb_versions,+ resolve_storage_dir_for_rebuild,+ write_version_meta,+)+++def test_retry_retains_source_identity_and_verified_parse_receipt(tmp_path):+ source = tmp_path / "raw" / "book.pdf"+ source.parent.mkdir()+ source.write_bytes(b"original textbook")+ with IndexingRun(tmp_path, [str(source)], task_id="first") as run:+ run.document(source, "parsed", parse_reused=True)+ run.finish("failed", error=TimeoutError("embedding timed out"))+ first = load_run(tmp_path)+ assert first["error_code"] == "timeout"+ assert first["documents"]["book.pdf"]["status"] == "unknown"+ with IndexingRun(tmp_path, [str(source)], task_id="retry") as run:+ assert run.data["previous_task_id"] == "first"+ assert (+ run.data["documents"]["book.pdf"]["source_hash"]+ == first["documents"]["book.pdf"]["source_hash"]+ )+ run.document(source, "completed", parse_reused=True)+ run.finish("completed", version="version-2")+ assert visible_run(tmp_path)["usable_version"] == "version-2"+ assert current_run() is None+++def test_cancel_is_owned_by_task_and_preserves_newer_worker(tmp_path):+ with IndexingRun(tmp_path, [], task_id="one") as run:+ assert not request_cancel(tmp_path, "other")+ assert request_cancel(tmp_path, "one")+ with pytest.raises(IndexingCancelled):+ run.check()+ run.finish("cancelled")+ with IndexingRun(tmp_path, [], task_id="two") as run:+ run.check()+ assert not request_cancel(tmp_path, "one")+ run.finish("completed")+++def test_live_owner_cannot_be_replaced_and_corrupt_journal_survives(tmp_path):+ with IndexingRun(tmp_path, [], task_id="one") as run:+ with pytest.raises(IndexingBusyError):+ with IndexingRun(tmp_path, [], task_id="two"):+ pass+ run.finish("completed")+ path = tmp_path / ".indexing-run.json"+ path.write_text("{broken")+ with pytest.raises(OSError):+ with IndexingRun(tmp_path, []):+ pass+ assert path.read_text() == "{broken"+++def test_restart_projection_is_read_only_and_retains_completed_sources(tmp_path):+ with IndexingRun(tmp_path, [], task_id="crashed") as run:+ run.finish("completed")+ path = tmp_path / ".indexing-run.json"+ data = json.loads(path.read_text())+ data.update(+ state="running",+ pid=999999999,+ documents={"a.pdf": {"status": "completed"}, "b.pdf": {"status": "embedding"}},+ )+ path.write_text(json.dumps(data))+ original = path.read_bytes()+ result = visible_run(tmp_path)+ assert result["state"] == "interrupted"+ assert result["documents"]["a.pdf"]["status"] == "completed"+ assert result["documents"]["b.pdf"]["status"] == "unknown"+ assert path.read_bytes() == original+++def test_changed_source_cannot_be_published(tmp_path):+ source = tmp_path / "raw" / "book.txt"+ source.parent.mkdir()+ source.write_text("before")+ with IndexingRun(tmp_path, [str(source)]) as run:+ run.document(source, "embedding")+ source.write_text("after")+ with pytest.raises(RuntimeError, match="Source changed"):+ run.verify_sources()+ run.finish("failed")+++def test_incomplete_version_is_not_queryable_and_previous_version_survives(tmp_path):+ signature = EmbeddingSignature("openai", "model", 3, "https://example.test", "")+ old = resolve_storage_dir_for_rebuild(tmp_path, signature, publication_guard=True)+ (old / "docstore.json").write_text("{}")+ write_version_meta(tmp_path, signature, old)+ candidate = resolve_storage_dir_for_rebuild(tmp_path, signature, publication_guard=True)+ (candidate / "docstore.json").write_text("partial")+ from deeptutor.services.rag.index_probe import inspect_provider_index++ assert not inspect_provider_index("llamaindex", candidate).ready+ versions = list_kb_versions(tmp_path)+ assert next(row for row in versions if row["version"] == old.name)["ready"]+ assert not next(row for row in versions if row["version"] == candidate.name)["ready"]+++def test_failure_details_redact_quoted_credentials(tmp_path):+ with IndexingRun(tmp_path, []) as run:+ run.finish("failed", error=RuntimeError('error {"api_key": "secret"} Bearer abc123'))+ detail = load_run(tmp_path)["error"]+ assert "secret" not in detail and "abc123" not in detail+++def test_real_persisted_index_reloads_and_missing_vectors_fail(tmp_path, monkeypatch):+ from llama_index.core import Document, Settings, VectorStoreIndex+ from llama_index.core.embeddings import MockEmbedding++ from deeptutor.services.rag.pipelines.llamaindex import storage++ monkeypatch.setattr(Settings, "_embed_model", MockEmbedding(embed_dim=4))+ index = VectorStoreIndex.from_documents(+ [Document(text="A known source about Fourier transforms.")]+ )+ index.storage_context.persist(persist_dir=str(tmp_path))+ storage.verify_persisted_index(tmp_path)+ (tmp_path / "default__vector_store.json").unlink()+ with pytest.raises(Exception):+ storage.verify_persisted_index(tmp_path)+++async def test_readiness_is_local_and_cancel_requires_writable_owned_task(tmp_path, monkeypatch):+ from fastapi import HTTPException++ from deeptutor.api.routers import knowledge+ from deeptutor.services.embedding import config+ from deeptutor.services.parsing import base+ from deeptutor.services.parsing.engines import factory++ resource = SimpleNamespace(name="kb", base_dir=tmp_path)+ monkeypatch.setattr(knowledge, "resolve_kb", lambda name: resource)+ monkeypatch.setattr(+ factory,+ "get_parser",+ lambda name: SimpleNamespace(+ resolve_config=lambda: SimpleNamespace(mode="local", device="cpu"),+ is_ready=lambda config: base.ReadinessReport(+ False, "models_missing", "Download models explicitly."+ ),+ supported_formats=lambda: {".pdf"},+ ),+ )+ monkeypatch.setattr(config, "get_embedding_config", lambda: SimpleNamespace(model="known"))+ result = await knowledge.get_indexing_readiness("kb")+ assert not result["ready"] and result["reason"] == "models_missing"+ assert result["device"] == "cpu"+ assert not (tmp_path / "kb").exists()++ def reject(name):+ raise HTTPException(403, "read only")++ monkeypatch.setattr(knowledge, "_writable_kb", reject)+ with pytest.raises(HTTPException) as error:+ await knowledge.cancel_indexing_run("kb", "someone-else")+ assert error.value.status_code == 403diff --git a/tests/services/rag/test_embedding_binding.py b/tests/services/rag/test_embedding_binding.pyindex fff505b9c..52ee5c913 100644--- a/tests/services/rag/test_embedding_binding.py+++ b/tests/services/rag/test_embedding_binding.py@@ -447,7 +447,14 @@ async def test_missing_binding_returns_actionable_search_error(catalog, tmp_path@pytest.mark.asyncio-async def test_unidentified_legacy_graph_index_keeps_its_existing_runtime(catalog, tmp_path):+async def test_unidentified_legacy_graph_index_keeps_its_existing_runtime(+ catalog, tmp_path, monkeypatch+):+ from deeptutor.services.rag import index_probe++ monkeypatch.setattr(+ index_probe, "latest_ready_provider_version", lambda *args: {"version": "version-1"}+ )from deeptutor.services.embedding.config import scoped_embedding_configwrite_entry(@@ -528,7 +535,7 @@ async def test_real_llamaindex_create_search_append_and_rebuild_use_bound_model(assert read_entry(tmp_path)["embedding_signature"] == originalassert await service.initialize("kb", ["first", "second"], embedding_selection=selection("b"))assert read_entry(tmp_path)["embedding_selection"] == selection("b")- assert len(list((tmp_path / "kb").glob("version-*"))) == 2+ assert len(list((tmp_path / "kb").glob("version-*"))) == 3assert (await service.search("study", "kb"))["sources"]clients.reset_embedding_client()clear_index_cache()diff --git a/tests/services/rag/test_llamaindex_pipeline_stall.py b/tests/services/rag/test_llamaindex_pipeline_stall.pyindex 9e4add08f..71bac9ad7 100644--- a/tests/services/rag/test_llamaindex_pipeline_stall.py+++ b/tests/services/rag/test_llamaindex_pipeline_stall.py@@ -338,6 +338,7 @@ async def test_initialize_succeeds_when_indexing_completes(_pipeline_module, storage_module, _ = _llamaindex_modules()pipeline = _make_pipeline(tmp_path, monkeypatch)monkeypatch.setattr(storage_module, "create_index", lambda *a, **k: 7)+ monkeypatch.setattr(storage_module, "verify_persisted_index", lambda path: None)assert await pipeline.initialize("kb", ["doc.pdf"]) is Truediff --git a/tests/services/rag/test_llamaindex_storage_layout.py b/tests/services/rag/test_llamaindex_storage_layout.pyindex 59870a866..f9dadcfd4 100644--- a/tests/services/rag/test_llamaindex_storage_layout.py+++ b/tests/services/rag/test_llamaindex_storage_layout.py@@ -46,6 +46,7 @@ async def test_reindex_receipt_excludes_a_file_skipped_by_parsing(monkeypatch.setattr(LlamaIndexPipeline, "_verify_embedding_connectivity", verify_embedding)monkeypatch.setattr(pipeline_module, "set_progress_callback", lambda _callback: None)monkeypatch.setattr(storage_module, "create_index", create_index)+ monkeypatch.setattr(storage_module, "verify_persisted_index", lambda path: None)pipeline = LlamaIndexPipeline(kb_base_dir=str(tmp_path), signature_provider=_signature)receipts: list[list[str]] = []@@ -114,6 +115,7 @@ async def test_incremental_add_migrates_matching_legacy_index_to_flat_version(_verify_embedding_connectivity,)monkeypatch.setattr(storage_module.vector_store, "load_index", _fake_load_index)+ monkeypatch.setattr(storage_module, "verify_persisted_index", lambda path: None)pipeline = LlamaIndexPipeline(kb_base_dir=str(tmp_path),diff --git a/tests/services/rag/test_rag_pipelines.py b/tests/services/rag/test_rag_pipelines.pyindex 9c4ed2455..87a58a981 100644--- a/tests/services/rag/test_rag_pipelines.py+++ b/tests/services/rag/test_rag_pipelines.py@@ -43,7 +43,12 @@ class _FakePipeline:@pytest.fixture-def fake_service(tmp_path) -> tuple[RAGService, _FakePipeline]:+def fake_service(tmp_path, monkeypatch) -> tuple[RAGService, _FakePipeline]:+ from deeptutor.services.rag import index_probe++ monkeypatch.setattr(+ index_probe, "latest_ready_provider_version", lambda *args: {"version": "fake"}+ )pipeline = _FakePipeline()service = RAGService(kb_base_dir=str(tmp_path))# No metadata.json under tmp_path → KBs resolve to the default provider.diff --git a/web/components/knowledge/KbIndexingRun.tsx b/web/components/knowledge/KbIndexingRun.tsxnew file mode 100644index 000000000..e560ff173--- /dev/null+++ b/web/components/knowledge/KbIndexingRun.tsx@@ -0,0 +1,78 @@+"use client";++import { useEffect, useState } from "react";+import { useTranslation } from "react-i18next";+import { apiFetch } from "@/shared/api/client";++interface Run {+ task_id: string;+ state: string;+ phase: string;+ started_at: number;+ finished_at?: number;+ last_activity_at: number;+ last_progress_at: number;+ phase_current?: number;+ phase_total?: number;+ usable_version?: string;+ cancel_requested: boolean;+ recovery: string;+ error?: string;+ documents: Record<string, { status: string; reason?: string; detail?: string; parse_reused: boolean }>;+}++export default function KbIndexingRun({ kbName, readOnly = false }: { kbName: string; readOnly?: boolean }) {+ const { t } = useTranslation();+ const [open, setOpen] = useState(false);+ const [snapshot, setSnapshot] = useState<{ key: string; run: Run | null }>();+ const [error, setError] = useState(false);+ const [now, setNow] = useState(Date.now() / 1000);+ const [cancelling, setCancelling] = useState(false);+ const run = snapshot?.key === kbName ? snapshot.run : undefined;+ useEffect(() => {+ if (!open) return;+ const controller = new AbortController();+ let timer: ReturnType<typeof setTimeout>;+ const poll = async () => {+ try {+ const response = await apiFetch(`/api/knowledge-bases/${encodeURIComponent(kbName)}/indexing-run`, { signal: controller.signal, cache: "no-store" });+ if (!response.ok) throw new Error(String(response.status));+ const data = await response.json() as { run: Run | null };+ if (!controller.signal.aborted) {+ setSnapshot({ key: kbName, run: data.run });+ setError(false);+ setNow(Date.now() / 1000);+ }+ } catch { if (!controller.signal.aborted) setError(true); }+ if (!controller.signal.aborted) timer = setTimeout(() => void poll(), 3000);+ };+ void poll();+ return () => { controller.abort(); clearTimeout(timer); };+ }, [kbName, open]);+ const cancel = async () => {+ if (!run) return;+ setCancelling(true);+ try {+ const response = await apiFetch(`/api/knowledge-bases/${encodeURIComponent(kbName)}/indexing-run/${encodeURIComponent(run.task_id)}/cancel`, { method: "POST" });+ if (!response.ok) throw new Error(String(response.status));+ setSnapshot({ key: kbName, run: { ...run, cancel_requested: true } });+ } catch { setError(true); }+ finally { setCancelling(false); }+ };+ return <details open={open} className="shrink-0 border-b border-[var(--border)] px-6 py-2 text-xs">+ <summary className="cursor-pointer" onClick={(event) => { event.preventDefault(); setOpen((value) => !value); }}>{t("Indexing recovery details")}</summary>+ {open && <div className="max-h-56 space-y-2 overflow-auto py-2">+ {error && <p role="alert">{t("Could not read indexing state. Refresh and retry.")}</p>}+ {run ? <>+ <p>{t("Run state")}: {t(`indexingState.${run.state}`, { defaultValue: run.state })} · {t("Current phase")}: {t(`indexingPhase.${run.phase}`, { defaultValue: run.phase })}</p>+ <p>{t("Runtime")}: {Math.max(0, Math.floor((run.finished_at || now) - run.started_at))}s · {t("Since activity")}: {Math.max(0, Math.floor(now - run.last_activity_at))}s · {t("Since measurable progress")}: {Math.max(0, Math.floor(now - run.last_progress_at))}s</p>+ <p>{run.phase_total ? `${run.phase_current || 0}/${run.phase_total}` : t("Phase progress is indeterminate.")} {t("Phase completion does not mean the whole index is ready.")}</p>+ {run.usable_version && <p>{t("Verified index version")}: {run.usable_version}</p>}+ <p>{t("Retry reuses validated parser outputs. Interrupted embedding stages restart.")}</p>+ <ul>{Object.entries(run.documents).map(([name, doc]) => <li className="break-words" key={name}>{name}: {t(`indexingDocument.${doc.status}`, { defaultValue: doc.status })}{doc.parse_reused ? ` · ${t("Parse cache reused")}` : ""}{doc.reason ? ` · ${doc.reason}` : ""}{doc.detail ? ` · ${doc.detail}` : ""}</li>)}</ul>+ {run.error && <p role="alert">{run.error}</p>}+ {!readOnly && run.state === "running" && <button className="underline" disabled={cancelling || run.cancel_requested} onClick={() => void cancel()}>{run.cancel_requested ? t("Cancellation pending at a safe boundary") : t("Cancel indexing")}</button>}+ </> : <p>{run === null ? t("No durable indexing run recorded yet.") : t("Loading...")}</p>}+ </div>}+ </details>;+}diff --git a/web/components/knowledge/KbTaskLogs.tsx b/web/components/knowledge/KbTaskLogs.tsxindex 18a721518..88e12c947 100644--- a/web/components/knowledge/KbTaskLogs.tsx+++ b/web/components/knowledge/KbTaskLogs.tsx@@ -56,7 +56,7 @@ export default function KbTaskLogs({/></div><span className="text-[11px] text-[var(--muted-foreground)]">- {percent}%+ {t("Current phase")}: {percent}%</span></div>)}diff --git a/web/components/knowledge/KnowledgeBaseDetail.tsx b/web/components/knowledge/KnowledgeBaseDetail.tsxindex 79315e720..8c79ae53e 100644--- a/web/components/knowledge/KnowledgeBaseDetail.tsx+++ b/web/components/knowledge/KnowledgeBaseDetail.tsx@@ -42,6 +42,7 @@ import type { TaskState } from "@/hooks/useKnowledgeProgress";import type { HistoryEntry } from "@/hooks/useKnowledgeHistory";import KbStatusBadge from "./KbStatusBadge";import KbTaskLogs from "./KbTaskLogs";+import KbIndexingRun from "./KbIndexingRun";import KbFilesTab from "./KbFilesTab";import KbDocumentsSection from "./KbDocumentsSection";import KbIndexVersionsSection from "./KbIndexVersionsSection";@@ -311,6 +312,7 @@ export default function KnowledgeBaseDetail({{/* Body */}<KbTaskLogs key={knowledgeBaseRef(kb)} kb={kb} task={task} />+ <KbIndexingRun key={`run-${knowledgeBaseRef(kb)}`} kbName={knowledgeBaseRef(kb)} readOnly={kb.read_only} /><div className="min-h-0 flex-1 overflow-hidden">{activeSection === "files" ? (<KbFilesTab key={knowledgeBaseRef(kb)} kb={kb} task={task} />diff --git a/web/locales/de/app.json b/web/locales/de/app.jsonindex 410c675b7..bfa1c7874 100644--- a/web/locales/de/app.json+++ b/web/locales/de/app.json@@ -5487,5 +5487,20 @@"Independent visual evidence is still required for this objective.": "Für dieses Lernziel ist noch ein eigenständiger visueller Nachweis erforderlich.","Assisted practice is recorded separately from independent mastery.": "Angeleitete Übung wird getrennt von eigenständiger Beherrschung erfasst.","Please review assessment {{questionId}} against its original source evidence. If it is unsupported, invalidate it and recompute learning state. Do not grant mastery merely because I challenged it.": "Bitte prüfe Bewertung {{questionId}} anhand der Originalquelle. Wenn sie unbelegt ist, annulliere sie und berechne den Lernstand neu. Vergib keinen Beherrschungsnachweis allein wegen meines Einspruchs.",- "Current model input has no verified source pixels. Retrieve the image again with a vision-capable model before grading.": "In der aktuellen Modelleingabe sind keine verifizierten Quellbilder enthalten. Lade das Bild vor der Bewertung mit einem visuellen Modell erneut."+ "Current model input has no verified source pixels. Retrieve the image again with a vision-capable model before grading.": "In der aktuellen Modelleingabe sind keine verifizierten Quellbilder enthalten. Lade das Bild vor der Bewertung mit einem visuellen Modell erneut.",+ "Indexing recovery details": "Details zur Indexwiederherstellung",+ "Could not read indexing state. Refresh and retry.": "Indexstatus konnte nicht gelesen werden. Aktualisieren und erneut versuchen.",+ "Run state": "Laufstatus",+ "Current phase": "Aktuelle Phase",+ "Runtime": "Laufzeit",+ "Since activity": "Seit letzter Aktivität",+ "Since measurable progress": "Seit messbarem Fortschritt",+ "Phase progress is indeterminate.": "Phasenfortschritt ist unbestimmt.",+ "Phase completion does not mean the whole index is ready.": "Eine abgeschlossene Phase bedeutet nicht, dass der gesamte Index bereit ist.",+ "Verified index version": "Geprüfte Indexversion",+ "Retry reuses validated parser outputs. Interrupted embedding stages restart.": "Wiederholung verwendet geprüfte Parserausgaben. Unterbrochene Einbettungsphasen starten neu.",+ "Parse cache reused": "Parsercache wiederverwendet",+ "Cancellation pending at a safe boundary": "Abbruch wartet auf sichere Grenze",+ "Cancel indexing": "Indexierung abbrechen",+ "No durable indexing run recorded yet.": "Noch kein dauerhafter Indexierungslauf aufgezeichnet."}diff --git a/web/locales/en/app.json b/web/locales/en/app.jsonindex 869989547..6b94ea2ee 100644--- a/web/locales/en/app.json+++ b/web/locales/en/app.json@@ -5487,5 +5487,20 @@"Independent visual evidence is still required for this objective.": "Independent visual evidence is still required for this objective.","Assisted practice is recorded separately from independent mastery.": "Assisted practice is recorded separately from independent mastery.","Please review assessment {{questionId}} against its original source evidence. If it is unsupported, invalidate it and recompute learning state. Do not grant mastery merely because I challenged it.": "Please review assessment {{questionId}} against its original source evidence. If it is unsupported, invalidate it and recompute learning state. Do not grant mastery merely because I challenged it.",- "Current model input has no verified source pixels. Retrieve the image again with a vision-capable model before grading.": "Current model input has no verified source pixels. Retrieve the image again with a vision-capable model before grading."+ "Current model input has no verified source pixels. Retrieve the image again with a vision-capable model before grading.": "Current model input has no verified source pixels. Retrieve the image again with a vision-capable model before grading.",+ "Indexing recovery details": "Indexing recovery details",+ "Could not read indexing state. Refresh and retry.": "Could not read indexing state. Refresh and retry.",+ "Run state": "Run state",+ "Current phase": "Current phase",+ "Runtime": "Runtime",+ "Since activity": "Since activity",+ "Since measurable progress": "Since measurable progress",+ "Phase progress is indeterminate.": "Phase progress is indeterminate.",+ "Phase completion does not mean the whole index is ready.": "Phase completion does not mean the whole index is ready.",+ "Verified index version": "Verified index version",+ "Retry reuses validated parser outputs. Interrupted embedding stages restart.": "Retry reuses validated parser outputs. Interrupted embedding stages restart.",+ "Parse cache reused": "Parse cache reused",+ "Cancellation pending at a safe boundary": "Cancellation pending at a safe boundary",+ "Cancel indexing": "Cancel indexing",+ "No durable indexing run recorded yet.": "No durable indexing run recorded yet."}diff --git a/web/locales/pl/app.json b/web/locales/pl/app.jsonindex f89767b9f..146e4ff3c 100644--- a/web/locales/pl/app.json+++ b/web/locales/pl/app.json@@ -5503,5 +5503,20 @@"Independent visual evidence is still required for this objective.": "Ten cel nadal wymaga samodzielnego dowodu rozumienia obrazu.","Assisted practice is recorded separately from independent mastery.": "Ćwiczenia ze wsparciem są zapisywane oddzielnie od samodzielnego opanowania.","Please review assessment {{questionId}} against its original source evidence. If it is unsupported, invalidate it and recompute learning state. Do not grant mastery merely because I challenged it.": "Sprawdź ocenę {{questionId}} na podstawie oryginalnego źródła. Jeśli jest nieuzasadniona, unieważnij ją i przelicz stan nauki. Samo zakwestionowanie nie dowodzi opanowania.",- "Current model input has no verified source pixels. Retrieve the image again with a vision-capable model before grading.": "Bieżące dane wejściowe modelu nie zawierają zweryfikowanego obrazu źródłowego. Przed oceną pobierz go ponownie modelem obsługującym obrazy."+ "Current model input has no verified source pixels. Retrieve the image again with a vision-capable model before grading.": "Bieżące dane wejściowe modelu nie zawierają zweryfikowanego obrazu źródłowego. Przed oceną pobierz go ponownie modelem obsługującym obrazy.",+ "Indexing recovery details": "Szczegóły odzyskiwania indeksu",+ "Could not read indexing state. Refresh and retry.": "Nie można odczytać stanu indeksowania. Odśwież i spróbuj ponownie.",+ "Run state": "Stan zadania",+ "Current phase": "Bieżący etap",+ "Runtime": "Czas działania",+ "Since activity": "Od ostatniej aktywności",+ "Since measurable progress": "Od mierzalnego postępu",+ "Phase progress is indeterminate.": "Postęp etapu jest nieokreślony.",+ "Phase completion does not mean the whole index is ready.": "Ukończenie etapu nie oznacza gotowości całego indeksu.",+ "Verified index version": "Zweryfikowana wersja indeksu",+ "Retry reuses validated parser outputs. Interrupted embedding stages restart.": "Ponowienie wykorzystuje zweryfikowane wyniki parsera. Przerwane etapy osadzania zaczynają się ponownie.",+ "Parse cache reused": "Wykorzystano pamięć podręczną parsera",+ "Cancellation pending at a safe boundary": "Anulowanie oczekuje na bezpieczną granicę",+ "Cancel indexing": "Anuluj indeksowanie",+ "No durable indexing run recorded yet.": "Nie zapisano jeszcze trwałego zadania indeksowania."}diff --git a/web/locales/zh/app.json b/web/locales/zh/app.jsonindex a0232ad34..57cbada1d 100644--- a/web/locales/zh/app.json+++ b/web/locales/zh/app.json@@ -5487,5 +5487,20 @@"Independent visual evidence is still required for this objective.": "此目标仍需要独立视觉证据。","Assisted practice is recorded separately from independent mastery.": "辅助练习与独立掌握分开记录。","Please review assessment {{questionId}} against its original source evidence. If it is unsupported, invalidate it and recompute learning state. Do not grant mastery merely because I challenged it.": "请对照原始来源复核判定 {{questionId}}。如果没有可靠依据,请作废并重算学习状态;不能只因我提出申诉就授予掌握状态。",- "Current model input has no verified source pixels. Retrieve the image again with a vision-capable model before grading.": "当前模型输入没有已核验的来源像素。请使用支持视觉的模型重新读取原图,再进行评分。"+ "Current model input has no verified source pixels. Retrieve the image again with a vision-capable model before grading.": "当前模型输入没有已核验的来源像素。请使用支持视觉的模型重新读取原图,再进行评分。",+ "Indexing recovery details": "索引恢复详情",+ "Could not read indexing state. Refresh and retry.": "无法读取索引状态,请刷新重试。",+ "Run state": "任务状态",+ "Current phase": "当前阶段",+ "Runtime": "运行时长",+ "Since activity": "距最近活动",+ "Since measurable progress": "距最近可测进度",+ "Phase progress is indeterminate.": "当前阶段进度无法确定。",+ "Phase completion does not mean the whole index is ready.": "阶段完成不代表整个索引已经就绪。",+ "Verified index version": "已验证索引版本",+ "Retry reuses validated parser outputs. Interrupted embedding stages restart.": "重试复用已验证的解析结果。中断的嵌入阶段将重新执行。",+ "Parse cache reused": "已复用解析缓存",+ "Cancellation pending at a safe boundary": "等待安全边界取消",+ "Cancel indexing": "取消索引",+ "No durable indexing run recorded yet.": "尚未记录持久索引任务。"}diff --git a/web/tests/kb-indexing-run.spec.tsx b/web/tests/kb-indexing-run.spec.tsxnew file mode 100644index 000000000..e0c336129--- /dev/null+++ b/web/tests/kb-indexing-run.spec.tsx@@ -0,0 +1,37 @@+import React from "react";+import { fireEvent, render, screen } from "@testing-library/react";+import { beforeEach, expect, it, vi } from "vitest";+import KbIndexingRun from "@/components/knowledge/KbIndexingRun";+import { apiFetch } from "@/shared/api/client";++vi.mock("react-i18next", () => ({ useTranslation: () => ({ t: (key: string) => key }) }));+vi.mock("@/shared/api/client", () => ({ apiFetch: vi.fn() }));+const run = { task_id: "owned", state: "running", phase: "embedding", started_at: 1, last_activity_at: 2,+ last_progress_at: 2, phase_current: 1, phase_total: 1, cancel_requested: false,+ documents: { "book.pdf": { status: "unknown", parse_reused: true }, "bad.pdf": { status: "failed", reason: "invalid_output" } } };+beforeEach(() => vi.clearAllMocks());+it("reads durable outcomes on demand and distinguishes phase completion from readiness", async () => {+ vi.mocked(apiFetch).mockResolvedValue(new Response(JSON.stringify({ run })));+ render(<KbIndexingRun kbName="kb" />);+ expect(apiFetch).not.toHaveBeenCalled();+ fireEvent.click(screen.getByText("Indexing recovery details"));+ await screen.findByText(/book.pdf/);+ expect(screen.getByText(/bad.pdf/)).toHaveTextContent("invalid_output");+ expect(screen.getByText(/Phase completion does not mean/)).toBeInTheDocument();+ expect(screen.queryByText(/Verified index version/)).not.toBeInTheDocument();+});+it("requests cancellation for the displayed owner and keeps it pending until a safe boundary", async () => {+ vi.mocked(apiFetch).mockResolvedValueOnce(new Response(JSON.stringify({ run }))).mockResolvedValueOnce(new Response("{}"));+ render(<KbIndexingRun kbName="kb" />);+ fireEvent.click(screen.getByText("Indexing recovery details"));+ fireEvent.click(await screen.findByRole("button", { name: "Cancel indexing" }));+ expect(await screen.findByRole("button", { name: "Cancellation pending at a safe boundary" })).toBeDisabled();+ expect(vi.mocked(apiFetch).mock.calls[1][0]).toContain("/indexing-run/owned/cancel");+});+it("read-only libraries have no cancellation action", async () => {+ vi.mocked(apiFetch).mockResolvedValue(new Response(JSON.stringify({ run })));+ render(<KbIndexingRun kbName="library:kb" readOnly />);+ fireEvent.click(screen.getByText("Indexing recovery details"));+ await screen.findByText(/book.pdf/);+ expect(screen.queryByRole("button", { name: "Cancel indexing" })).not.toBeInTheDocument();+});