Skip to content

DeepTutor: Lifelong Personalized Tutoring. https://deeptutor.info/.

fix(knowledge): make long-document indexing recoverable and publication verified (#1612)

Bingxi Zhao (Frank)e79e1b0ad2a9
diff --git a/deeptutor/api/routers/knowledge.py b/deeptutor/api/routers/knowledge.py
index e8fd777a6..d9234b30b 100644
--- a/deeptutor/api/routers/knowledge.py
+++ b/deeptutor/api/routers/knowledge.py
@@ -1023,6 +1023,13 @@ async def run_initialization_task(
indexed_count = len(
FileTypeRouter.collect_supported_files(initializer.raw_dir, recursive=True)
)
+ from deeptutor.knowledge.indexing_run import load_run
+
+ receipt = load_run(initializer.raw_dir.parent)
+ if receipt and receipt.get("provider") == "llamaindex":
+ indexed_count = sum(
+ doc.get("status") == "completed" for doc in receipt["documents"].values()
+ )
initializer.progress_tracker.update(
ProgressStage.COMPLETED,
@@ -3994,6 +4001,7 @@ async def run_reindex_task(
manager._save_config()
success = await rag_service.initialize(
+ task_id=task_id,
kb_name=kb_name,
file_paths=file_paths,
progress_callback=_on_progress,
@@ -4106,6 +4114,20 @@ async def reindex_knowledge_base(
kb_entry = _load_kb_entry_or_404(manager, kb_name)
_assert_not_connected_kb(kb_name, kb_entry)
force_reindex = str(kb_entry.get("status") or "").lower() == "error"
+ from deeptutor.knowledge.indexing_run import visible_run
+
+ previous_run = visible_run(kb_base_dir / kb_name)
+ if previous_run:
+ if previous_run["state"] == "running":
+ raise HTTPException(
+ status_code=409, detail="An indexing worker still owns this knowledge base."
+ )
+ force_reindex = force_reindex or previous_run["state"] in {
+ "partial",
+ "failed",
+ "cancelled",
+ "interrupted",
+ }
kb_provider = _validate_registered_provider(
kb_entry.get("rag_provider") or DEFAULT_PROVIDER
)
@@ -4301,6 +4323,59 @@ async def retry_knowledge_base(
raise HTTPException(status_code=500, detail=format_exception_message(e))
[email protected]("/knowledge-bases/{kb_name}/indexing-run")
+async def get_indexing_run(kb_name: str):
+ """Pure durable diagnostics; reading never recreates a task or retries work."""
+ from deeptutor.knowledge.indexing_run import visible_run
+
+ resource = resolve_kb(kb_name)
+ return {"run": visible_run(resource.base_dir / resource.name)}
+
+
[email protected]("/knowledge-bases/{kb_name}/indexing-run/{task_id}/cancel")
+async def cancel_indexing_run(kb_name: str, task_id: str):
+ from deeptutor.knowledge.indexing_run import request_cancel
+
+ _, name, base_dir = _writable_kb(kb_name)
+ if not request_cancel(base_dir / name, task_id):
+ raise HTTPException(status_code=409, detail="This indexing run is no longer active.")
+ return {
+ "cancel_requested": True,
+ "message": "Cancellation takes effect at the next safe boundary; completed parser outputs remain reusable.",
+ }
+
+
[email protected]("/knowledge-bases/{kb_name}/indexing-readiness")
+async def get_indexing_readiness(kb_name: str):
+ from deeptutor.services.embedding.config import get_embedding_config
+ from deeptutor.services.parsing import get_parse_service
+ from deeptutor.services.parsing.engines.factory import get_parser
+
+ resource = resolve_kb(kb_name)
+ engine = get_parse_service().active_engine()
+ parser = get_parser(engine)
+ config = parser.resolve_config()
+ report = parser.is_ready(config)
+ embedding = get_embedding_config()
+ return {
+ "parser": engine,
+ "ready": report.ready,
+ "reason": report.reason,
+ "message": report.message,
+ "formats": sorted(parser.supported_formats()),
+ "mode": getattr(config, "mode", None),
+ "device": getattr(config, "device", None),
+ "embedding_model": embedding.model,
+ "embedding_configured": bool(embedding.model),
+ "source_count": len(
+ FileTypeRouter.collect_supported_files(
+ resource.base_dir / resource.name / "raw", recursive=True
+ )
+ ),
+ "note": "Readiness checks configuration and local prerequisites; runtime downloads, resource limits and provider responses can still fail.",
+ }
+
+
@router.get("/knowledge-bases/{kb_name}/progress")
async def get_progress(kb_name: str):
"""Get initialization progress for a knowledge base"""
diff --git a/deeptutor/knowledge/indexing_run.py b/deeptutor/knowledge/indexing_run.py
new file mode 100644
index 000000000..43ce47016
--- /dev/null
+++ b/deeptutor/knowledge/indexing_run.py
@@ -0,0 +1,447 @@
+"""Durable per-KB indexing state and ownership at reliable boundaries (#1612).
+
+Parsing caches remain the reusable work store. This journal records what was
+accepted, parsed, failed, published, or unknown after interruption; it is not a
+queue or a second parsing cache.
+"""
+
+from __future__ import annotations
+
+import asyncio
+from contextvars import ContextVar
+from hashlib import sha256
+import json
+import os
+from pathlib import Path
+import re
+import sys
+import threading
+import time
+from typing import Any
+from uuid import uuid4
+
+try:
+ import psutil
+except ImportError: # CLI-only installs need no process-monitor dependency.
+ psutil = None
+
+from deeptutor.services.file_io import atomic_write_json
+
+_CURRENT: ContextVar[IndexingRun | None] = ContextVar("indexing_run", default=None)
+_TERMINAL = {"completed", "partial", "failed", "cancelled", "interrupted"}
+
+
+class IndexingBusyError(RuntimeError):
+ pass
+
+
+class IndexingCancelled(RuntimeError):
+ pass
+
+
+def current_run() -> IndexingRun | None:
+ return _CURRENT.get()
+
+
+def _redact(detail: str) -> str:
+ text = re.sub(r"(?i)(bearer\s+)[^\s]+", r"\1[redacted]", str(detail))
+ text = re.sub(
+ r"(?i)((?:api[_-]?key|token|secret|password|signature|authorization)\s*[=:]\s*)[^\s&]+",
+ r"\1[redacted]",
+ text,
+ )
+ text = re.sub(
+ r'(?i)(["\']?(?:api[_-]?key|token|secret|password|signature|authorization)["\']?\s*[:=]\s*)["\'][^"\']*["\']',
+ r'\1"[redacted]"',
+ text,
+ )
+ text = re.sub(r"(https?://)[^/\s@]+:[^/\s@]+@", r"\1[redacted]@", text)
+ return text[-1600:]
+
+
+def failure_code(exc: BaseException) -> str:
+ text = str(exc).casefold()
+ if isinstance(exc, (TimeoutError,)) or "timeout" in text or "timed out" in text:
+ return "timeout"
+ if isinstance(exc, MemoryError) or "out of memory" in text or "resource exhausted" in text:
+ return "resource_exhaustion"
+ if (
+ isinstance(exc, (FileNotFoundError, ModuleNotFoundError))
+ or "not installed" in text
+ or "cli_missing" in text
+ ):
+ return "dependency_missing"
+ if "unsupported" in text or "doesn't support" in text:
+ return "unsupported_input"
+ if "download" in text:
+ return "model_download_failed"
+ if "embedding" in text or "vector" in text:
+ return "embedding_failed"
+ if "no content" in text or "no output" in text or "invalid" in text:
+ return "invalid_output"
+ return "runtime_failure"
+
+
+def load_run(kb_dir: Path) -> dict[str, Any] | None:
+ path = Path(kb_dir) / ".indexing-run.json"
+ try:
+ value = json.loads(path.read_text())
+ except FileNotFoundError:
+ return None
+ except (OSError, ValueError) as exc:
+ raise OSError("Indexing journal is unreadable; original file preserved.") from exc
+ if (
+ not isinstance(value, dict)
+ or value.get("schema") != 1
+ or not isinstance(value.get("documents"), dict)
+ ):
+ raise OSError("Indexing journal is invalid; original file preserved.")
+ try:
+ cancelled = json.loads((Path(kb_dir) / ".indexing-cancel.json").read_text())
+ except FileNotFoundError:
+ cancelled = {}
+ except (OSError, ValueError) as exc:
+ raise OSError(
+ "Indexing cancellation state is unreadable; original file preserved."
+ ) from exc
+ value["cancel_requested"] = bool(
+ isinstance(cancelled, dict) and cancelled.get("task_id") == value.get("task_id")
+ )
+ return value
+
+
+def _lease_is_held(kb_dir: Path) -> bool:
+ try:
+ with (Path(kb_dir) / ".indexing.lock").open("r+b") as lease:
+ if sys.platform == "win32":
+ import msvcrt
+
+ msvcrt.locking(lease.fileno(), msvcrt.LK_NBLCK, 1)
+ msvcrt.locking(lease.fileno(), msvcrt.LK_UNLCK, 1)
+ else:
+ import fcntl
+
+ fcntl.flock(lease.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB)
+ fcntl.flock(lease.fileno(), fcntl.LOCK_UN)
+ return False
+ except FileNotFoundError:
+ return False
+ except OSError:
+ return True
+
+
+def visible_run(kb_dir: Path) -> dict[str, Any] | None:
+ """Pure projection: a crashed owner's in-flight work becomes unknown."""
+ value = load_run(kb_dir)
+ if value is None or value.get("state") in _TERMINAL:
+ return value
+ try:
+ alive = (
+ psutil.Process(int(value["pid"])).create_time() == value["process_started_at"]
+ if psutil is not None
+ else _lease_is_held(kb_dir)
+ )
+ except Exception:
+ alive = False
+ if alive:
+ return value
+ documents = {
+ key: {
+ **doc,
+ "status": "unknown"
+ if doc.get("status") in {"pending", "parsing", "embedding"}
+ else doc.get("status"),
+ }
+ for key, doc in value["documents"].items()
+ }
+ return {
+ **value,
+ "state": "interrupted",
+ "documents": documents,
+ "recovery": "Retry reuses validated parser outputs and completed slices. The interrupted uncheckpointed stage restarts.",
+ }
+
+
+class IndexingRun:
+ def __init__(
+ self,
+ kb_dir: Path,
+ sources: list[str],
+ *,
+ task_id: str = "",
+ action: str = "rebuild",
+ provider: str = "",
+ reporter=None,
+ ):
+ self.kb_dir = Path(kb_dir)
+ self.path = self.kb_dir / ".indexing-run.json"
+ self.task_id = task_id or f"index_{uuid4().hex}"
+ self.action = action
+ self.provider = provider
+ self.sources = sources
+ self.reporter = reporter
+ self._mutex = threading.RLock()
+ self.active = False
+ self.data: dict[str, Any] = {}
+ self._lock = None
+ self._token = None
+ self._source_paths: dict[str, Path] = {}
+
+ def __enter__(self):
+ return self._start()
+
+ async def __aenter__(self):
+ future = asyncio.create_task(asyncio.to_thread(self._start, False))
+ try:
+ await asyncio.shield(future)
+ except asyncio.CancelledError:
+ await future
+ self.finish("cancelled")
+ self.active = False
+ self._release()
+ raise
+ self._token = _CURRENT.set(self)
+ return self
+
+ async def __aexit__(self, kind, exc, traceback):
+ self.__exit__(kind, exc, traceback)
+
+ def _start(self, bind_context=True):
+ self.kb_dir.mkdir(parents=True, exist_ok=True)
+ self._lock = (self.kb_dir / ".indexing.lock").open("a+b")
+ try:
+ if sys.platform == "win32":
+ import msvcrt
+
+ self._lock.seek(0)
+ if not self._lock.read(1):
+ self._lock.write(b"\0")
+ self._lock.flush()
+ self._lock.seek(0)
+ msvcrt.locking(self._lock.fileno(), msvcrt.LK_NBLCK, 1)
+ else:
+ import fcntl
+
+ fcntl.flock(self._lock.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB)
+ except OSError as exc:
+ self._lock.close()
+ raise IndexingBusyError(
+ "An indexing worker still owns this knowledge base. Wait for it to settle before retrying."
+ ) from exc
+ try:
+ previous = visible_run(self.kb_dir)
+ now = time.time()
+ self.data = {
+ "schema": 1,
+ "task_id": self.task_id,
+ "action": self.action,
+ "provider": self.provider,
+ "pid": os.getpid(),
+ "process_started_at": psutil.Process().create_time()
+ if psutil is not None
+ else None,
+ "state": "running",
+ "phase": "preflight",
+ "started_at": now,
+ "last_activity_at": now,
+ "last_progress_at": now,
+ "documents": {},
+ "previous_task_id": previous.get("task_id") if previous else None,
+ "recovery": "Validated parser caches and completed slices are reused. Uncheckpointed stages restart.",
+ "cancel_requested": False,
+ "usable_version": None,
+ }
+ from deeptutor.services.rag.file_routing import FileTypeRouter
+
+ candidates = list(dict.fromkeys(self.sources))
+ raw = self.kb_dir / "raw"
+ if self.action != "upload" and raw.is_dir():
+ candidates.extend(
+ str(path)
+ for path in raw.rglob("*")
+ if path.is_file() and not path.is_symlink() and str(path) not in candidates
+ )
+ classification = FileTypeRouter.classify_files(candidates)
+ unsupported = set(classification.unsupported)
+ for name in candidates:
+ source = Path(name)
+ key = self.source_key(source)
+ self._source_paths[key] = source
+ try:
+ digest = self._hash(source)
+ reason = "unsupported_input" if name in unsupported else ""
+ status = "excluded" if reason else "pending"
+ except OSError as exc:
+ digest, reason, status = "", _redact(str(exc)), "failed"
+ self.data["documents"][key] = {
+ "status": status,
+ "reason": reason,
+ "source_hash": digest,
+ "parse_reused": False,
+ }
+ self.active = True
+ if bind_context:
+ self._token = _CURRENT.set(self)
+ self._save()
+ return self
+ except BaseException:
+ self.active = False
+ if self._token is not None:
+ _CURRENT.reset(self._token)
+ self._release()
+ raise
+
+ def __exit__(self, kind, exc, traceback):
+ try:
+ if exc is not None:
+ self.finish(
+ "partial"
+ if self.data.get("usable_version")
+ else "cancelled"
+ if isinstance(exc, IndexingCancelled)
+ or (load_run(self.kb_dir) or {}).get("cancel_requested")
+ else "failed",
+ version=self.data.get("usable_version"),
+ error=exc,
+ )
+ elif self.data.get("state") not in _TERMINAL:
+ self.finish("failed", error=RuntimeError("Index readiness was not verified."))
+ finally:
+ self.active = False
+ if self._token is not None:
+ _CURRENT.reset(self._token)
+ self._release()
+
+ def _release(self):
+ if self._lock is not None and not self._lock.closed:
+ if sys.platform == "win32":
+ import msvcrt
+
+ self._lock.seek(0)
+ msvcrt.locking(self._lock.fileno(), msvcrt.LK_UNLCK, 1)
+ else:
+ import fcntl
+
+ fcntl.flock(self._lock.fileno(), fcntl.LOCK_UN)
+ self._lock.close()
+
+ def source_key(self, source: Path) -> str:
+ try:
+ return source.resolve().relative_to((self.kb_dir / "raw").resolve()).as_posix()
+ except ValueError:
+ return (
+ f"external-{sha256(str(source.resolve()).encode()).hexdigest()[:12]}/{source.name}"
+ )
+
+ @staticmethod
+ def _hash(source: Path) -> str:
+ digest = sha256()
+ with source.open("rb") as stream:
+ for chunk in iter(lambda: stream.read(1024 * 1024), b""):
+ digest.update(chunk)
+ return digest.hexdigest()
+
+ def check(self):
+ if not self.active:
+ raise IndexingCancelled("This indexing worker no longer owns the run.")
+ stored = load_run(self.kb_dir)
+ if stored and stored.get("task_id") != self.task_id:
+ raise IndexingCancelled("A newer run owns the indexing journal.")
+ if stored and stored.get("cancel_requested"):
+ self.data["cancel_requested"] = True
+ raise IndexingCancelled(
+ "Cancellation reached a safe boundary; validated completed parsing remains reusable."
+ )
+
+ def phase(
+ self,
+ phase: str,
+ *,
+ source: Path | None = None,
+ current: int | None = None,
+ total: int | None = None,
+ activity: str = "",
+ ):
+ with self._mutex:
+ self.check()
+ now = time.time()
+ changed = self.data.get("phase") != phase or (
+ current is not None and current != self.data.get("phase_current")
+ )
+ self.data.update(
+ {
+ "phase": phase,
+ "last_activity_at": now,
+ "current_document": self.source_key(source)
+ if source
+ else self.data.get("current_document"),
+ "phase_current": current,
+ "phase_total": total,
+ "activity": _redact(activity),
+ }
+ )
+ if changed:
+ self.data["last_progress_at"] = now
+ self._save()
+
+ def document(self, source: Path, status: str, **details):
+ with self._mutex:
+ if not self.active:
+ return
+ stored = load_run(self.kb_dir)
+ if stored and stored.get("task_id") != self.task_id:
+ return
+ key = self.source_key(source)
+ doc = self.data["documents"].setdefault(key, {})
+ doc.update({"status": status, "updated_at": time.time(), **details})
+ self.data["last_progress_at"] = time.time()
+ self._save()
+
+ def verify_sources(self):
+ self.check()
+ for key, doc in self.data["documents"].items():
+ if doc.get("status") in {"parsed", "embedding", "completed"}:
+ if self._hash(self._source_paths[key]) != doc.get("source_hash"):
+ raise RuntimeError(
+ f"Source changed during indexing: {key}. Retry with the current source."
+ )
+
+ def finish(self, state: str, *, version: str | None = None, error: BaseException | None = None):
+ with self._mutex:
+ if not self.active:
+ return
+ stored = load_run(self.kb_dir)
+ if stored and stored.get("task_id") != self.task_id:
+ return
+ for doc in self.data["documents"].values():
+ if doc.get("status") in {"pending", "parsing", "embedding", "parsed"}:
+ doc["status"] = "unknown"
+ self.data.update(
+ {
+ "state": state,
+ "finished_at": time.time(),
+ "usable_version": version,
+ "error_code": failure_code(error) if error else None,
+ "error": _redact(str(error)) if error else None,
+ }
+ )
+ self._save()
+
+ def _save(self):
+ atomic_write_json(self.path, self.data)
+ if self.reporter is not None:
+ try:
+ self.reporter(dict(self.data))
+ except Exception:
+ # Notification delivery cannot invalidate the durable checkpoint.
+ pass
+
+
+def request_cancel(kb_dir: Path, task_id: str) -> bool:
+ value = load_run(kb_dir)
+ if value is None or value.get("task_id") != task_id or value.get("state") in _TERMINAL:
+ return False
+ atomic_write_json(
+ Path(kb_dir) / ".indexing-cancel.json", {"task_id": task_id, "requested_at": time.time()}
+ )
+ return True
diff --git a/deeptutor/knowledge/initializer.py b/deeptutor/knowledge/initializer.py
index 74ee212f3..364bd2ba3 100644
--- a/deeptutor/knowledge/initializer.py
+++ b/deeptutor/knowledge/initializer.py
@@ -107,6 +107,15 @@ class KnowledgeBaseInitializer:
metadata["last_indexed_count"] = len(
FileTypeRouter.collect_supported_files(self.raw_dir, recursive=True)
)
+ if provider == "llamaindex":
+ from deeptutor.knowledge.indexing_run import load_run
+
+ receipt = load_run(self.kb_dir)
+ if receipt:
+ metadata["last_indexed_count"] = sum(
+ doc.get("status") == "completed" for doc in receipt["documents"].values()
+ )
+ metadata["indexing_state"] = receipt["state"]
metadata["last_indexed_action"] = "create"
atomic_write_json(metadata_file, metadata)
@@ -238,6 +247,7 @@ class KnowledgeBaseInitializer:
success = await rag_service.initialize(
kb_name=self.kb_name,
file_paths=file_paths,
+ task_id=self.progress_tracker.task_id,
progress_callback=_on_progress,
image_progress_callback=_on_image_progress,
indexing_snapshot=self.indexing_snapshot,
diff --git a/deeptutor/services/parsing/engines/mineru/backend.py b/deeptutor/services/parsing/engines/mineru/backend.py
index b0a0035c4..4f81dfb93 100644
--- a/deeptutor/services/parsing/engines/mineru/backend.py
+++ b/deeptutor/services/parsing/engines/mineru/backend.py
@@ -35,6 +35,8 @@ _LOCAL_CLI_COMMANDS = ("mineru", "magic-pdf")
#: hint is deliberately limited to ``CLI_MISSING``: a runtime failure must not
#: be reported as a missing installation (issue #1612, outcome 4).
_LOCAL_FAILURE_MESSAGES: dict[LocalParseReason, str] = {
+ LocalParseReason.TIMEOUT: "Local MinerU parsing timed out: {detail}",
+ LocalParseReason.CANCELLED: "Local MinerU parsing was cancelled: {detail}",
LocalParseReason.CLI_MISSING: (
"Local MinerU parsing failed. Ensure MinerU is installed "
"(`pip install -U 'mineru[all]>=3.4.5'`) or switch to cloud mode in "
diff --git a/deeptutor/services/parsing/engines/mineru/local.py b/deeptutor/services/parsing/engines/mineru/local.py
index 236d3fa7f..f3861b535 100644
--- a/deeptutor/services/parsing/engines/mineru/local.py
+++ b/deeptutor/services/parsing/engines/mineru/local.py
@@ -12,6 +12,7 @@ import shutil
import subprocess
import sys
import tempfile
+import threading
import time
from uuid import uuid4
@@ -24,6 +25,8 @@ from .formats import MINERU_SUPPORTED_FORMATS
# progress that universal-newline decoding turns into many lines per second;
# without a floor the trace panel gets flooded during model downloads.
_ON_OUTPUT_MIN_INTERVAL = 0.5
+_LOCAL_PARSE_IDLE_TIMEOUT_SECONDS = 600
+_LOCAL_PARSE_TIMEOUT_SECONDS = 7200
#: Upper bound on the failure excerpt carried back to callers: long enough for
#: a useful stderr tail, short enough to fit inside an error message.
@@ -49,6 +52,8 @@ class LocalParseReason(StrEnum):
INPUT_MISSING = "input_missing"
UNSUPPORTED_INPUT = "unsupported_input"
LEGACY_CLI_INPUT = "legacy_cli_input"
+ TIMEOUT = "timeout"
+ CANCELLED = "cancelled"
NONZERO_EXIT = "nonzero_exit"
NO_ARTIFACTS = "no_artifacts"
EXCEPTION = "exception"
@@ -204,6 +209,9 @@ def parse_document_with_mineru_result(
atomic_write_json(state_path, {"source": source_file.name, "state": "running"})
process = None
process_finished = False
+ watchdog_stop = threading.Event()
+ watchdog = None
+ interrupted: list[LocalParseReason] = []
result = LocalParseResult.failure(LocalParseReason.EXCEPTION, "parse interrupted")
try:
cmd = [mineru_cmd, "-p", str(source_file), "-o", str(temp_output)]
@@ -225,11 +233,44 @@ def parse_document_with_mineru_result(
)
tail: deque[str] = deque(maxlen=40)
last_emit = 0.0
+ from deeptutor.knowledge.indexing_run import IndexingCancelled, current_run
+
+ run = current_run()
+ activity = [time.monotonic()]
+ if run is not None:
+ started = activity[0]
+
+ def watch():
+ while not watchdog_stop.wait(0.25):
+ try:
+ run.check()
+ except IndexingCancelled:
+ interrupted.append(LocalParseReason.CANCELLED)
+ except OSError:
+ interrupted.append(LocalParseReason.EXCEPTION)
+ if (
+ time.monotonic() - activity[0] > _LOCAL_PARSE_IDLE_TIMEOUT_SECONDS
+ or time.monotonic() - started > _LOCAL_PARSE_TIMEOUT_SECONDS
+ ):
+ interrupted.append(LocalParseReason.TIMEOUT)
+ if interrupted:
+ try:
+ process.terminate()
+ process.wait(timeout=3)
+ except subprocess.TimeoutExpired:
+ process.kill()
+ except OSError:
+ pass
+ return
+
+ watchdog = threading.Thread(target=watch, daemon=True)
+ watchdog.start()
assert process.stdout is not None
for raw_line in process.stdout:
line = raw_line.strip()
if not line:
continue
+ activity[0] = time.monotonic()
tail.append(line)
if on_output is not None:
now = time.monotonic()
@@ -244,6 +285,12 @@ def parse_document_with_mineru_result(
returncode = process.wait()
process_finished = True
+ if interrupted:
+ result = LocalParseResult.failure(
+ interrupted[0],
+ "Cancellation or parse time limit reached; completed checkpoints remain reusable.",
+ )
+ return result
if returncode != 0:
print("✗ MinerU parsing failed:")
print("\n".join(tail))
@@ -309,6 +356,9 @@ def parse_document_with_mineru_result(
)
return result
finally:
+ watchdog_stop.set()
+ if watchdog is not None:
+ watchdog.join(timeout=4)
# Stop an interrupted child before another attempt can publish output.
if process is not None and not process_finished:
try:
diff --git a/deeptutor/services/parsing/service.py b/deeptutor/services/parsing/service.py
index 2db211b24..5834ea086 100644
--- a/deeptutor/services/parsing/service.py
+++ b/deeptutor/services/parsing/service.py
@@ -101,7 +101,35 @@ class ParseService:
supported = get_parser(engine_name).supported_formats()
return not supported or _matches_supported_format(source_path, supported)
- def parse(
+ def parse(self, source_path: str | Path, *, engine=None, on_output=None) -> ParsedDocument:
+ from deeptutor.knowledge.indexing_run import _redact, current_run, failure_code
+
+ run = current_run()
+ source = Path(source_path)
+ if run:
+ run.phase("parsing", source=source)
+ run.document(source, "parsing")
+
+ def output(line):
+ if run:
+ run.phase("parsing", source=source, activity=line)
+ if on_output:
+ on_output(line)
+
+ try:
+ result = self._parse(source, engine=engine, on_output=output)
+ if run:
+ run.check()
+ run.document(
+ source, "parsed", parser=result.engine, parser_signature=result.parser_signature
+ )
+ return result
+ except Exception as exc:
+ if run:
+ run.document(source, "failed", reason=failure_code(exc), detail=_redact(str(exc)))
+ raise
+
+ def _parse(
self,
source_path: str | Path,
*,
@@ -161,6 +189,10 @@ class ParseService:
markdown, blocks, asset_dir = "", None, None
if markdown.strip() or blocks:
logger.info("Parse cache hit for %s (%s/%s)", source_path.name, engine_name, sig)
+ from deeptutor.knowledge.indexing_run import current_run
+
+ if run := current_run():
+ run.document(source_path, "parsed", parse_reused=True)
return ParsedDocument(
markdown=markdown,
blocks=blocks,
diff --git a/deeptutor/services/rag/index_probe.py b/deeptutor/services/rag/index_probe.py
index c55458e4b..a2a5704ce 100644
--- a/deeptutor/services/rag/index_probe.py
+++ b/deeptutor/services/rag/index_probe.py
@@ -143,6 +143,15 @@ def _inspect_llamaindex(storage_dir: Path) -> ProviderIndexProbe:
"LlamaIndex storage directory does not exist.",
)
+ if (storage_dir / ".building.json").exists():
+ meta = _read_json(storage_dir / "meta.json")
+ if not meta or meta.get("state") != "published" or not meta.get("readiness_verified"):
+ return ProviderIndexProbe(
+ DEFAULT_PROVIDER,
+ str(storage_dir),
+ False,
+ "Candidate index has not passed persistence verification.",
+ )
docstore = storage_dir / "docstore.json"
index_store = storage_dir / "index_store.json"
vector_stores = sorted(path.name for path in storage_dir.glob("*vector_store.json"))
diff --git a/deeptutor/services/rag/index_versioning.py b/deeptutor/services/rag/index_versioning.py
index 0e50ba367..644e314d2 100644
--- a/deeptutor/services/rag/index_versioning.py
+++ b/deeptutor/services/rag/index_versioning.py
@@ -153,6 +153,8 @@ def _entry_from_flat_version(version_dir: Path) -> dict[str, Any]:
or (version_dir / "deeptutor_ingress").exists()
)
ready = has_provider_output and not is_unpublished_lightrag_candidate
+ if (version_dir / ".building.json").exists():
+ ready = ready and bool(stored_meta and stored_meta.get("state") == "published")
if stored_meta is not None and stored_meta.get("provider") == "lightrag":
adapter_schema = stored_meta.get("lightrag_adapter_schema")
if adapter_schema is not None:
@@ -317,8 +319,11 @@ def write_version_meta(
"layout": "flat" if target.parent == kb_dir else "nested_legacy",
"created_at": datetime.now(timezone.utc).replace(tzinfo=None).isoformat() + "Z",
}
- with open(target / META_FILENAME, "w", encoding="utf-8") as handle:
- json.dump(payload, handle, indent=2, ensure_ascii=False)
+ from deeptutor.services.file_io import atomic_write_json
+
+ if (target / ".building.json").exists():
+ payload.update(provider="llamaindex", state="published", readiness_verified=True)
+ atomic_write_json(target / META_FILENAME, payload)
def resolve_storage_dir_for_read(
@@ -356,7 +361,9 @@ def resolve_storage_dir_for_write(kb_dir: Path, signature: Optional[EmbeddingSig
return target
-def resolve_storage_dir_for_rebuild(kb_dir: Path, signature: Optional[EmbeddingSignature]) -> Path:
+def resolve_storage_dir_for_rebuild(
+ kb_dir: Path, signature: Optional[EmbeddingSignature], *, publication_guard: bool = False
+) -> Path:
"""Return a fresh flat storage dir for a full index rebuild.
Incremental writes reuse a matching flat version, but a full rebuild should
@@ -365,6 +372,16 @@ def resolve_storage_dir_for_rebuild(kb_dir: Path, signature: Optional[EmbeddingS
rebuild remain diagnosable and avoids stale vector-store files.
"""
_ = signature
- target = _next_flat_version_dir(kb_dir)
- target.mkdir(parents=True, exist_ok=True)
+ kb_dir.mkdir(parents=True, exist_ok=True)
+ while True:
+ target = _next_flat_version_dir(kb_dir)
+ try:
+ target.mkdir()
+ break
+ except FileExistsError:
+ continue
+ from deeptutor.services.file_io import atomic_write_json
+
+ if publication_guard:
+ atomic_write_json(target / ".building.json", {"state": "building"})
return target
diff --git a/deeptutor/services/rag/pipelines/llamaindex/document_loader.py b/deeptutor/services/rag/pipelines/llamaindex/document_loader.py
index 26f4d5625..103488727 100644
--- a/deeptutor/services/rag/pipelines/llamaindex/document_loader.py
+++ b/deeptutor/services/rag/pipelines/llamaindex/document_loader.py
@@ -172,6 +172,30 @@ class LlamaIndexDocumentLoader:
for file_path_str in classification.unsupported:
self.logger.warning(f"Skipped unsupported file: {Path(file_path_str).name}")
+ from deeptutor.knowledge.indexing_run import current_run
+
+ if run := current_run():
+ accepted = {
+ str(Path(doc.metadata["file_path"]).resolve())
+ for doc in documents
+ if isinstance(getattr(doc, "metadata", None), dict)
+ and doc.metadata.get("file_path")
+ }
+ for name in (
+ classification.parser_files + classification.text_files + classification.image_files
+ ):
+ source = Path(name)
+ if str(source.resolve()) in accepted:
+ run.document(source, "embedding")
+ elif (
+ run.data["documents"].get(run.source_key(source), {}).get("status") != "failed"
+ ):
+ run.document(
+ source,
+ "failed",
+ reason="invalid_output",
+ detail="No usable document content.",
+ )
return documents
def _parse_document(
diff --git a/deeptutor/services/rag/pipelines/llamaindex/pipeline.py b/deeptutor/services/rag/pipelines/llamaindex/pipeline.py
index fe280e9e8..798a1b466 100644
--- a/deeptutor/services/rag/pipelines/llamaindex/pipeline.py
+++ b/deeptutor/services/rag/pipelines/llamaindex/pipeline.py
@@ -209,7 +209,7 @@ class LlamaIndexPipeline:
)
signature = self._current_signature()
- storage_dir = resolve_storage_dir_for_rebuild(kb_dir, signature)
+ storage_dir = resolve_storage_dir_for_rebuild(kb_dir, signature, publication_guard=True)
try:
await self._verify_embedding_connectivity()
@@ -242,8 +242,7 @@ class LlamaIndexPipeline:
)
self.logger.info(f"Index persisted to {storage_dir}")
- if signature is not None:
- write_version_meta(kb_dir, signature, storage_dir=storage_dir)
+ await self._publish_verified(kb_dir, storage_dir, signature, documents)
indexed_file_callback = kwargs.get("indexed_file_callback")
if indexed_file_callback is not None:
@@ -437,8 +436,7 @@ class LlamaIndexPipeline:
worker_key=str(kb_dir.resolve()),
)
self.logger.info(f"Added {num_added} documents to existing index")
- if signature is not None and plan.storage_dir != plan.existing_storage:
- write_version_meta(kb_dir, signature, storage_dir=plan.storage_dir)
+
else:
self.logger.info(f"Creating new index with {len(documents)} documents...")
plan.storage_dir.mkdir(parents=True, exist_ok=True)
@@ -450,9 +448,8 @@ class LlamaIndexPipeline:
worker_key=str(kb_dir.resolve()),
)
self.logger.info(f"Created new index with {num_added} documents")
- if signature is not None:
- write_version_meta(kb_dir, signature, storage_dir=plan.storage_dir)
+ await self._publish_verified(kb_dir, plan.storage_dir, signature, documents)
self.logger.info(f"Successfully added documents to KB '{kb_name}'")
return True
@@ -467,6 +464,35 @@ class LlamaIndexPipeline:
finally:
set_progress_callback(None)
+ async def _publish_verified(self, kb_dir, storage_dir, signature, documents):
+ from deeptutor.knowledge.indexing_run import current_run
+ from deeptutor.services.file_io import atomic_write_json
+
+ run = current_run()
+ if run:
+ run.phase("verifying")
+ await asyncio.to_thread(storage.verify_persisted_index, storage_dir)
+ if run:
+ await asyncio.to_thread(run.verify_sources)
+ if signature is not None:
+ write_version_meta(kb_dir, signature, storage_dir=storage_dir)
+ else:
+ atomic_write_json(
+ storage_dir / "meta.json",
+ {
+ "version": storage_dir.name,
+ "provider": "llamaindex",
+ "state": "published",
+ "readiness_verified": True,
+ },
+ )
+ if run:
+ run.data["usable_version"] = storage_dir.name
+ for document in documents:
+ path = (getattr(document, "metadata", None) or {}).get("file_path")
+ if path:
+ run.document(Path(path), "completed")
+
async def delete(self, kb_name: str) -> bool:
kb_dir = resolve_kb_dir(self.kb_base_dir, kb_name)
deleted = storage.delete_kb_dir(kb_dir)
diff --git a/deeptutor/services/rag/pipelines/llamaindex/storage.py b/deeptutor/services/rag/pipelines/llamaindex/storage.py
index 4dfb406ea..499176304 100644
--- a/deeptutor/services/rag/pipelines/llamaindex/storage.py
+++ b/deeptutor/services/rag/pipelines/llamaindex/storage.py
@@ -15,9 +15,8 @@ from typing import Any
from deeptutor.services.embedding.validation import validate_embedding_batch
from deeptutor.services.rag.index_versioning import (
EmbeddingSignature,
- find_matching_version,
resolve_storage_dir_for_read,
- resolve_storage_dir_for_write,
+ resolve_storage_dir_for_rebuild,
)
from . import ingestion, retrievers, vector_store
@@ -59,32 +58,12 @@ def cleanup_failed_version_dir(storage_dir: Path) -> bool:
def resolve_add_storage_plan(kb_dir: Path, signature: EmbeddingSignature | None) -> AddStoragePlan:
"""Choose existing/new storage dirs for incremental adds."""
- matching_version = find_matching_version(kb_dir, signature) if signature is not None else None
- existing_storage = (
- _storage_path_from_version_entry(matching_version) if matching_version else None
+ existing = resolve_storage_dir_for_read(kb_dir, signature)
+ return AddStoragePlan(
+ existing_storage=existing,
+ storage_dir=resolve_storage_dir_for_rebuild(kb_dir, signature, publication_guard=True),
)
- if matching_version and existing_storage and matching_version.get("layout") == "flat":
- return AddStoragePlan(existing_storage=existing_storage, storage_dir=existing_storage)
-
- if matching_version and existing_storage:
- return AddStoragePlan(
- existing_storage=existing_storage,
- storage_dir=resolve_storage_dir_for_write(kb_dir, signature),
- )
-
- fallback_storage = resolve_storage_dir_for_read(kb_dir, signature)
- existing_storage = fallback_storage
- fallback_is_flat = (
- fallback_storage is not None
- and fallback_storage.parent == kb_dir
- and fallback_storage.name.startswith("version-")
- )
- storage_dir = (
- fallback_storage if fallback_is_flat else resolve_storage_dir_for_write(kb_dir, signature)
- )
- return AddStoragePlan(existing_storage=existing_storage, storage_dir=storage_dir)
-
def create_index(
documents: list[Any], storage_dir: Path, *, show_progress: bool | None = None
@@ -316,3 +295,26 @@ def delete_kb_dir(kb_dir: Path) -> bool:
shutil.rmtree(kb_dir)
return True
return False
+
+
+def verify_persisted_index(storage_dir: Path) -> None:
+ index = vector_store.load_index(storage_dir)
+ _validate_persisted_embeddings(index, storage_dir)
+ if not index.docstore.docs:
+ raise ValueError("Persisted index contains no retrievable nodes.")
+ if not getattr(index, "index_struct", None):
+ raise ValueError("Persisted index has no index structure.")
+ vectors = [values for _, values in _iter_index_embedding_dicts(index) if values]
+ if vectors:
+ sample = next(iter(vectors[0].values()))
+ else:
+ client = getattr(index.vector_store, "client", None)
+ if client is None or not getattr(client, "ntotal", 0):
+ raise ValueError("Persisted index contains no embedding vectors.")
+ sample = client.reconstruct(0).tolist()
+ from llama_index.core.schema import QueryBundle
+
+ if not index.as_retriever(similarity_top_k=1).retrieve(
+ QueryBundle(query_str="", embedding=sample)
+ ):
+ raise ValueError("Persisted index failed a retrieval probe with a stored vector.")
diff --git a/deeptutor/services/rag/service.py b/deeptutor/services/rag/service.py
index 5fb4a3050..49da9488d 100644
--- a/deeptutor/services/rag/service.py
+++ b/deeptutor/services/rag/service.py
@@ -66,21 +66,62 @@ class RAGService:
@with_kb_embedding
async def initialize(self, kb_name: str, file_paths: List[str], **kwargs) -> bool:
- provider = self._resolve_provider(kb_name)
- self.logger.info(f"Initializing KB '{kb_name}' (provider={provider})")
- pipeline = self._get_pipeline(provider)
- return await pipeline.initialize(kb_name=kb_name, file_paths=file_paths, **kwargs)
+ return await self._index(kb_name, file_paths, "initialize", kwargs)
@with_kb_embedding
async def add_documents(self, kb_name: str, file_paths: List[str], **kwargs) -> bool:
+ return await self._index(kb_name, file_paths, "add_documents", kwargs)
+
+ async def _index(self, kb_name, file_paths, method, kwargs):
+ from deeptutor.knowledge.indexing_run import IndexingRun, current_run
+
+ from .index_probe import latest_ready_provider_version
+ from .kb_paths import resolve_kb_dir
+
provider = self._resolve_provider(kb_name)
- self.logger.info(
- f"Adding {len(file_paths)} document(s) to KB '{kb_name}' (provider={provider})"
- )
pipeline = self._get_pipeline(provider)
- if not hasattr(pipeline, "add_documents"):
- return await pipeline.initialize(kb_name=kb_name, file_paths=file_paths, **kwargs)
- return await pipeline.add_documents(kb_name=kb_name, file_paths=file_paths, **kwargs)
+ operation = getattr(pipeline, method, None) or pipeline.initialize
+ if current_run() is not None:
+ return await operation(kb_name=kb_name, file_paths=file_paths, **kwargs)
+ task_id = kwargs.pop("task_id", "")
+ async with IndexingRun(
+ resolve_kb_dir(self.kb_base_dir, kb_name),
+ file_paths,
+ task_id=task_id,
+ action="upload" if method == "add_documents" else "rebuild",
+ provider=provider,
+ ) as run:
+ receipt_callback = kwargs.get("indexed_file_callback")
+
+ def indexed(paths):
+ for path in paths:
+ run.document(Path(path), "completed")
+ if receipt_callback:
+ receipt_callback(paths)
+
+ kwargs["indexed_file_callback"] = indexed
+ callback = kwargs.get("progress_callback")
+
+ def progress(current, total):
+ run.phase("embedding", current=current, total=total)
+ if callback:
+ callback(current, total)
+
+ kwargs["progress_callback"] = progress
+ success = await operation(kb_name=kb_name, file_paths=file_paths, **kwargs)
+ if not run.data.get("usable_version"):
+ run.check()
+ if not success:
+ return False
+ version = latest_ready_provider_version(run.kb_dir, provider)
+ if version is None:
+ raise RuntimeError("Index output did not pass the provider readiness check.")
+ partial = any(
+ doc.get("status") in {"failed", "unknown", "pending", "parsing"}
+ for doc in run.data["documents"].values()
+ )
+ run.finish("partial" if partial else "completed", version=version.get("version"))
+ return True
@with_kb_embedding
async def search(
diff --git a/docs/INDEXING_RECOVERY.md b/docs/INDEXING_RECOVERY.md
new file mode 100644
index 000000000..7e3e71117
--- /dev/null
+++ b/docs/INDEXING_RECOVERY.md
@@ -0,0 +1,37 @@
+# Long-document indexing and recovery
+
+Each knowledge base records its latest indexing run in `.indexing-run.json`.
+The Files/detail view exposes **Indexing recovery details**, including individual
+source outcomes, current phase and phase-local counts, elapsed runtime, time
+since output/activity, and time since measurable progress. A phase reaching its
+last batch is not proof that the whole index is ready. Unknown work remains
+unknown after interruption; it is never counted as a successful document.
+
+`GET /api/knowledge-bases/{id}/indexing-readiness` performs a lightweight parser,
+format and embedding-configuration check without downloading models or sending
+a provider request. Readiness does not guarantee later downloads, available RAM
+or GPU memory, nor a provider's runtime response. Runtime failures retain their
+bounded, credential-redacted reasons in the durable run.
+
+Retry uses the existing content-addressed parser cache: source bytes and parser
+signature must match, and usable text or blocks must still exist. MinerU cloud
+PDF slices reuse only validated checkpoints. Changing embeddings can reuse
+parsing, but builds a separate index version for the new embedding signature.
+Interrupted stages without a usable checkpoint restart; the journal itself is
+not a second cache and does not fabricate partial embeddings.
+
+Cancellation is task-specific and takes effect at a safe boundary. Managed
+local MinerU processes also observe cancellation and fail after 600 seconds
+without output or 7200 seconds total. Other synchronous engines cannot be
+interrupted arbitrarily; cancellation remains pending until they return. A
+stalled LlamaIndex executor is barred from overlapping same-process retries
+until it actually exits, and cannot publish its candidate after losing ownership.
+
+LlamaIndex rebuilds and incremental additions use fresh version directories.
+A candidate stays unpublished until its persisted index reloads, its vectors
+validate and a retrieval probe using a stored vector returns a node. Publication
+metadata is written atomically. Source changes during indexing reject publication.
+A failed or partial candidate does not overwrite a previous queryable version.
+Completed and failed sources are reported separately, including partial success.
+The durable journal is scoped to the active user/workspace knowledge base;
+linked/read-only knowledge bases cannot be cancelled through mutation routes.
diff --git a/tests/knowledge/test_indexing_run.py b/tests/knowledge/test_indexing_run.py
new file mode 100644
index 000000000..0ecfe6556
--- /dev/null
+++ b/tests/knowledge/test_indexing_run.py
@@ -0,0 +1,179 @@
+"""Durable outcomes, interruption, cancellation and publication (#1612)."""
+
+import json
+from pathlib import Path
+from types import SimpleNamespace
+
+import pytest
+
+from deeptutor.knowledge.indexing_run import (
+ IndexingBusyError,
+ IndexingCancelled,
+ IndexingRun,
+ current_run,
+ load_run,
+ request_cancel,
+ visible_run,
+)
+from deeptutor.services.rag.index_versioning import (
+ EmbeddingSignature,
+ list_kb_versions,
+ resolve_storage_dir_for_rebuild,
+ write_version_meta,
+)
+
+
+def test_retry_retains_source_identity_and_verified_parse_receipt(tmp_path):
+ source = tmp_path / "raw" / "book.pdf"
+ source.parent.mkdir()
+ source.write_bytes(b"original textbook")
+ with IndexingRun(tmp_path, [str(source)], task_id="first") as run:
+ run.document(source, "parsed", parse_reused=True)
+ run.finish("failed", error=TimeoutError("embedding timed out"))
+ first = load_run(tmp_path)
+ assert first["error_code"] == "timeout"
+ assert first["documents"]["book.pdf"]["status"] == "unknown"
+ with IndexingRun(tmp_path, [str(source)], task_id="retry") as run:
+ assert run.data["previous_task_id"] == "first"
+ assert (
+ run.data["documents"]["book.pdf"]["source_hash"]
+ == first["documents"]["book.pdf"]["source_hash"]
+ )
+ run.document(source, "completed", parse_reused=True)
+ run.finish("completed", version="version-2")
+ assert visible_run(tmp_path)["usable_version"] == "version-2"
+ assert current_run() is None
+
+
+def test_cancel_is_owned_by_task_and_preserves_newer_worker(tmp_path):
+ with IndexingRun(tmp_path, [], task_id="one") as run:
+ assert not request_cancel(tmp_path, "other")
+ assert request_cancel(tmp_path, "one")
+ with pytest.raises(IndexingCancelled):
+ run.check()
+ run.finish("cancelled")
+ with IndexingRun(tmp_path, [], task_id="two") as run:
+ run.check()
+ assert not request_cancel(tmp_path, "one")
+ run.finish("completed")
+
+
+def test_live_owner_cannot_be_replaced_and_corrupt_journal_survives(tmp_path):
+ with IndexingRun(tmp_path, [], task_id="one") as run:
+ with pytest.raises(IndexingBusyError):
+ with IndexingRun(tmp_path, [], task_id="two"):
+ pass
+ run.finish("completed")
+ path = tmp_path / ".indexing-run.json"
+ path.write_text("{broken")
+ with pytest.raises(OSError):
+ with IndexingRun(tmp_path, []):
+ pass
+ assert path.read_text() == "{broken"
+
+
+def test_restart_projection_is_read_only_and_retains_completed_sources(tmp_path):
+ with IndexingRun(tmp_path, [], task_id="crashed") as run:
+ run.finish("completed")
+ path = tmp_path / ".indexing-run.json"
+ data = json.loads(path.read_text())
+ data.update(
+ state="running",
+ pid=999999999,
+ documents={"a.pdf": {"status": "completed"}, "b.pdf": {"status": "embedding"}},
+ )
+ path.write_text(json.dumps(data))
+ original = path.read_bytes()
+ result = visible_run(tmp_path)
+ assert result["state"] == "interrupted"
+ assert result["documents"]["a.pdf"]["status"] == "completed"
+ assert result["documents"]["b.pdf"]["status"] == "unknown"
+ assert path.read_bytes() == original
+
+
+def test_changed_source_cannot_be_published(tmp_path):
+ source = tmp_path / "raw" / "book.txt"
+ source.parent.mkdir()
+ source.write_text("before")
+ with IndexingRun(tmp_path, [str(source)]) as run:
+ run.document(source, "embedding")
+ source.write_text("after")
+ with pytest.raises(RuntimeError, match="Source changed"):
+ run.verify_sources()
+ run.finish("failed")
+
+
+def test_incomplete_version_is_not_queryable_and_previous_version_survives(tmp_path):
+ signature = EmbeddingSignature("openai", "model", 3, "https://example.test", "")
+ old = resolve_storage_dir_for_rebuild(tmp_path, signature, publication_guard=True)
+ (old / "docstore.json").write_text("{}")
+ write_version_meta(tmp_path, signature, old)
+ candidate = resolve_storage_dir_for_rebuild(tmp_path, signature, publication_guard=True)
+ (candidate / "docstore.json").write_text("partial")
+ from deeptutor.services.rag.index_probe import inspect_provider_index
+
+ assert not inspect_provider_index("llamaindex", candidate).ready
+ versions = list_kb_versions(tmp_path)
+ assert next(row for row in versions if row["version"] == old.name)["ready"]
+ assert not next(row for row in versions if row["version"] == candidate.name)["ready"]
+
+
+def test_failure_details_redact_quoted_credentials(tmp_path):
+ with IndexingRun(tmp_path, []) as run:
+ run.finish("failed", error=RuntimeError('error {"api_key": "secret"} Bearer abc123'))
+ detail = load_run(tmp_path)["error"]
+ assert "secret" not in detail and "abc123" not in detail
+
+
+def test_real_persisted_index_reloads_and_missing_vectors_fail(tmp_path, monkeypatch):
+ from llama_index.core import Document, Settings, VectorStoreIndex
+ from llama_index.core.embeddings import MockEmbedding
+
+ from deeptutor.services.rag.pipelines.llamaindex import storage
+
+ monkeypatch.setattr(Settings, "_embed_model", MockEmbedding(embed_dim=4))
+ index = VectorStoreIndex.from_documents(
+ [Document(text="A known source about Fourier transforms.")]
+ )
+ index.storage_context.persist(persist_dir=str(tmp_path))
+ storage.verify_persisted_index(tmp_path)
+ (tmp_path / "default__vector_store.json").unlink()
+ with pytest.raises(Exception):
+ storage.verify_persisted_index(tmp_path)
+
+
+async def test_readiness_is_local_and_cancel_requires_writable_owned_task(tmp_path, monkeypatch):
+ from fastapi import HTTPException
+
+ from deeptutor.api.routers import knowledge
+ from deeptutor.services.embedding import config
+ from deeptutor.services.parsing import base
+ from deeptutor.services.parsing.engines import factory
+
+ resource = SimpleNamespace(name="kb", base_dir=tmp_path)
+ monkeypatch.setattr(knowledge, "resolve_kb", lambda name: resource)
+ monkeypatch.setattr(
+ factory,
+ "get_parser",
+ lambda name: SimpleNamespace(
+ resolve_config=lambda: SimpleNamespace(mode="local", device="cpu"),
+ is_ready=lambda config: base.ReadinessReport(
+ False, "models_missing", "Download models explicitly."
+ ),
+ supported_formats=lambda: {".pdf"},
+ ),
+ )
+ monkeypatch.setattr(config, "get_embedding_config", lambda: SimpleNamespace(model="known"))
+ result = await knowledge.get_indexing_readiness("kb")
+ assert not result["ready"] and result["reason"] == "models_missing"
+ assert result["device"] == "cpu"
+ assert not (tmp_path / "kb").exists()
+
+ def reject(name):
+ raise HTTPException(403, "read only")
+
+ monkeypatch.setattr(knowledge, "_writable_kb", reject)
+ with pytest.raises(HTTPException) as error:
+ await knowledge.cancel_indexing_run("kb", "someone-else")
+ assert error.value.status_code == 403
diff --git a/tests/services/rag/test_embedding_binding.py b/tests/services/rag/test_embedding_binding.py
index fff505b9c..52ee5c913 100644
--- a/tests/services/rag/test_embedding_binding.py
+++ b/tests/services/rag/test_embedding_binding.py
@@ -447,7 +447,14 @@ async def test_missing_binding_returns_actionable_search_error(catalog, tmp_path
@pytest.mark.asyncio
-async def test_unidentified_legacy_graph_index_keeps_its_existing_runtime(catalog, tmp_path):
+async def test_unidentified_legacy_graph_index_keeps_its_existing_runtime(
+ catalog, tmp_path, monkeypatch
+):
+ from deeptutor.services.rag import index_probe
+
+ monkeypatch.setattr(
+ index_probe, "latest_ready_provider_version", lambda *args: {"version": "version-1"}
+ )
from deeptutor.services.embedding.config import scoped_embedding_config
write_entry(
@@ -528,7 +535,7 @@ async def test_real_llamaindex_create_search_append_and_rebuild_use_bound_model(
assert read_entry(tmp_path)["embedding_signature"] == original
assert await service.initialize("kb", ["first", "second"], embedding_selection=selection("b"))
assert read_entry(tmp_path)["embedding_selection"] == selection("b")
- assert len(list((tmp_path / "kb").glob("version-*"))) == 2
+ assert len(list((tmp_path / "kb").glob("version-*"))) == 3
assert (await service.search("study", "kb"))["sources"]
clients.reset_embedding_client()
clear_index_cache()
diff --git a/tests/services/rag/test_llamaindex_pipeline_stall.py b/tests/services/rag/test_llamaindex_pipeline_stall.py
index 9e4add08f..71bac9ad7 100644
--- a/tests/services/rag/test_llamaindex_pipeline_stall.py
+++ b/tests/services/rag/test_llamaindex_pipeline_stall.py
@@ -338,6 +338,7 @@ async def test_initialize_succeeds_when_indexing_completes(
_pipeline_module, storage_module, _ = _llamaindex_modules()
pipeline = _make_pipeline(tmp_path, monkeypatch)
monkeypatch.setattr(storage_module, "create_index", lambda *a, **k: 7)
+ monkeypatch.setattr(storage_module, "verify_persisted_index", lambda path: None)
assert await pipeline.initialize("kb", ["doc.pdf"]) is True
diff --git a/tests/services/rag/test_llamaindex_storage_layout.py b/tests/services/rag/test_llamaindex_storage_layout.py
index 59870a866..f9dadcfd4 100644
--- a/tests/services/rag/test_llamaindex_storage_layout.py
+++ b/tests/services/rag/test_llamaindex_storage_layout.py
@@ -46,6 +46,7 @@ async def test_reindex_receipt_excludes_a_file_skipped_by_parsing(
monkeypatch.setattr(LlamaIndexPipeline, "_verify_embedding_connectivity", verify_embedding)
monkeypatch.setattr(pipeline_module, "set_progress_callback", lambda _callback: None)
monkeypatch.setattr(storage_module, "create_index", create_index)
+ monkeypatch.setattr(storage_module, "verify_persisted_index", lambda path: None)
pipeline = LlamaIndexPipeline(kb_base_dir=str(tmp_path), signature_provider=_signature)
receipts: list[list[str]] = []
@@ -114,6 +115,7 @@ async def test_incremental_add_migrates_matching_legacy_index_to_flat_version(
_verify_embedding_connectivity,
)
monkeypatch.setattr(storage_module.vector_store, "load_index", _fake_load_index)
+ monkeypatch.setattr(storage_module, "verify_persisted_index", lambda path: None)
pipeline = LlamaIndexPipeline(
kb_base_dir=str(tmp_path),
diff --git a/tests/services/rag/test_rag_pipelines.py b/tests/services/rag/test_rag_pipelines.py
index 9c4ed2455..87a58a981 100644
--- a/tests/services/rag/test_rag_pipelines.py
+++ b/tests/services/rag/test_rag_pipelines.py
@@ -43,7 +43,12 @@ class _FakePipeline:
@pytest.fixture
-def fake_service(tmp_path) -> tuple[RAGService, _FakePipeline]:
+def fake_service(tmp_path, monkeypatch) -> tuple[RAGService, _FakePipeline]:
+ from deeptutor.services.rag import index_probe
+
+ monkeypatch.setattr(
+ index_probe, "latest_ready_provider_version", lambda *args: {"version": "fake"}
+ )
pipeline = _FakePipeline()
service = RAGService(kb_base_dir=str(tmp_path))
# No metadata.json under tmp_path → KBs resolve to the default provider.
diff --git a/web/components/knowledge/KbIndexingRun.tsx b/web/components/knowledge/KbIndexingRun.tsx
new file mode 100644
index 000000000..e560ff173
--- /dev/null
+++ b/web/components/knowledge/KbIndexingRun.tsx
@@ -0,0 +1,78 @@
+"use client";
+
+import { useEffect, useState } from "react";
+import { useTranslation } from "react-i18next";
+import { apiFetch } from "@/shared/api/client";
+
+interface Run {
+ task_id: string;
+ state: string;
+ phase: string;
+ started_at: number;
+ finished_at?: number;
+ last_activity_at: number;
+ last_progress_at: number;
+ phase_current?: number;
+ phase_total?: number;
+ usable_version?: string;
+ cancel_requested: boolean;
+ recovery: string;
+ error?: string;
+ documents: Record<string, { status: string; reason?: string; detail?: string; parse_reused: boolean }>;
+}
+
+export default function KbIndexingRun({ kbName, readOnly = false }: { kbName: string; readOnly?: boolean }) {
+ const { t } = useTranslation();
+ const [open, setOpen] = useState(false);
+ const [snapshot, setSnapshot] = useState<{ key: string; run: Run | null }>();
+ const [error, setError] = useState(false);
+ const [now, setNow] = useState(Date.now() / 1000);
+ const [cancelling, setCancelling] = useState(false);
+ const run = snapshot?.key === kbName ? snapshot.run : undefined;
+ useEffect(() => {
+ if (!open) return;
+ const controller = new AbortController();
+ let timer: ReturnType<typeof setTimeout>;
+ const poll = async () => {
+ try {
+ const response = await apiFetch(`/api/knowledge-bases/${encodeURIComponent(kbName)}/indexing-run`, { signal: controller.signal, cache: "no-store" });
+ if (!response.ok) throw new Error(String(response.status));
+ const data = await response.json() as { run: Run | null };
+ if (!controller.signal.aborted) {
+ setSnapshot({ key: kbName, run: data.run });
+ setError(false);
+ setNow(Date.now() / 1000);
+ }
+ } catch { if (!controller.signal.aborted) setError(true); }
+ if (!controller.signal.aborted) timer = setTimeout(() => void poll(), 3000);
+ };
+ void poll();
+ return () => { controller.abort(); clearTimeout(timer); };
+ }, [kbName, open]);
+ const cancel = async () => {
+ if (!run) return;
+ setCancelling(true);
+ try {
+ const response = await apiFetch(`/api/knowledge-bases/${encodeURIComponent(kbName)}/indexing-run/${encodeURIComponent(run.task_id)}/cancel`, { method: "POST" });
+ if (!response.ok) throw new Error(String(response.status));
+ setSnapshot({ key: kbName, run: { ...run, cancel_requested: true } });
+ } catch { setError(true); }
+ finally { setCancelling(false); }
+ };
+ return <details open={open} className="shrink-0 border-b border-[var(--border)] px-6 py-2 text-xs">
+ <summary className="cursor-pointer" onClick={(event) => { event.preventDefault(); setOpen((value) => !value); }}>{t("Indexing recovery details")}</summary>
+ {open && <div className="max-h-56 space-y-2 overflow-auto py-2">
+ {error && <p role="alert">{t("Could not read indexing state. Refresh and retry.")}</p>}
+ {run ? <>
+ <p>{t("Run state")}: {t(`indexingState.${run.state}`, { defaultValue: run.state })} · {t("Current phase")}: {t(`indexingPhase.${run.phase}`, { defaultValue: run.phase })}</p>
+ <p>{t("Runtime")}: {Math.max(0, Math.floor((run.finished_at || now) - run.started_at))}s · {t("Since activity")}: {Math.max(0, Math.floor(now - run.last_activity_at))}s · {t("Since measurable progress")}: {Math.max(0, Math.floor(now - run.last_progress_at))}s</p>
+ <p>{run.phase_total ? `${run.phase_current || 0}/${run.phase_total}` : t("Phase progress is indeterminate.")} {t("Phase completion does not mean the whole index is ready.")}</p>
+ {run.usable_version && <p>{t("Verified index version")}: {run.usable_version}</p>}
+ <p>{t("Retry reuses validated parser outputs. Interrupted embedding stages restart.")}</p>
+ <ul>{Object.entries(run.documents).map(([name, doc]) => <li className="break-words" key={name}>{name}: {t(`indexingDocument.${doc.status}`, { defaultValue: doc.status })}{doc.parse_reused ? ` · ${t("Parse cache reused")}` : ""}{doc.reason ? ` · ${doc.reason}` : ""}{doc.detail ? ` · ${doc.detail}` : ""}</li>)}</ul>
+ {run.error && <p role="alert">{run.error}</p>}
+ {!readOnly && run.state === "running" && <button className="underline" disabled={cancelling || run.cancel_requested} onClick={() => void cancel()}>{run.cancel_requested ? t("Cancellation pending at a safe boundary") : t("Cancel indexing")}</button>}
+ </> : <p>{run === null ? t("No durable indexing run recorded yet.") : t("Loading...")}</p>}
+ </div>}
+ </details>;
+}
diff --git a/web/components/knowledge/KbTaskLogs.tsx b/web/components/knowledge/KbTaskLogs.tsx
index 18a721518..88e12c947 100644
--- a/web/components/knowledge/KbTaskLogs.tsx
+++ b/web/components/knowledge/KbTaskLogs.tsx
@@ -56,7 +56,7 @@ export default function KbTaskLogs({
/>
</div>
<span className="text-[11px] text-[var(--muted-foreground)]">
- {percent}%
+ {t("Current phase")}: {percent}%
</span>
</div>
)}
diff --git a/web/components/knowledge/KnowledgeBaseDetail.tsx b/web/components/knowledge/KnowledgeBaseDetail.tsx
index 79315e720..8c79ae53e 100644
--- a/web/components/knowledge/KnowledgeBaseDetail.tsx
+++ b/web/components/knowledge/KnowledgeBaseDetail.tsx
@@ -42,6 +42,7 @@ import type { TaskState } from "@/hooks/useKnowledgeProgress";
import type { HistoryEntry } from "@/hooks/useKnowledgeHistory";
import KbStatusBadge from "./KbStatusBadge";
import KbTaskLogs from "./KbTaskLogs";
+import KbIndexingRun from "./KbIndexingRun";
import KbFilesTab from "./KbFilesTab";
import KbDocumentsSection from "./KbDocumentsSection";
import KbIndexVersionsSection from "./KbIndexVersionsSection";
@@ -311,6 +312,7 @@ export default function KnowledgeBaseDetail({
{/* Body */}
<KbTaskLogs key={knowledgeBaseRef(kb)} kb={kb} task={task} />
+ <KbIndexingRun key={`run-${knowledgeBaseRef(kb)}`} kbName={knowledgeBaseRef(kb)} readOnly={kb.read_only} />
<div className="min-h-0 flex-1 overflow-hidden">
{activeSection === "files" ? (
<KbFilesTab key={knowledgeBaseRef(kb)} kb={kb} task={task} />
diff --git a/web/locales/de/app.json b/web/locales/de/app.json
index 410c675b7..bfa1c7874 100644
--- a/web/locales/de/app.json
+++ b/web/locales/de/app.json
@@ -5487,5 +5487,20 @@
"Independent visual evidence is still required for this objective.": "Für dieses Lernziel ist noch ein eigenständiger visueller Nachweis erforderlich.",
"Assisted practice is recorded separately from independent mastery.": "Angeleitete Übung wird getrennt von eigenständiger Beherrschung erfasst.",
"Please review assessment {{questionId}} against its original source evidence. If it is unsupported, invalidate it and recompute learning state. Do not grant mastery merely because I challenged it.": "Bitte prüfe Bewertung {{questionId}} anhand der Originalquelle. Wenn sie unbelegt ist, annulliere sie und berechne den Lernstand neu. Vergib keinen Beherrschungsnachweis allein wegen meines Einspruchs.",
- "Current model input has no verified source pixels. Retrieve the image again with a vision-capable model before grading.": "In der aktuellen Modelleingabe sind keine verifizierten Quellbilder enthalten. Lade das Bild vor der Bewertung mit einem visuellen Modell erneut."
+ "Current model input has no verified source pixels. Retrieve the image again with a vision-capable model before grading.": "In der aktuellen Modelleingabe sind keine verifizierten Quellbilder enthalten. Lade das Bild vor der Bewertung mit einem visuellen Modell erneut.",
+ "Indexing recovery details": "Details zur Indexwiederherstellung",
+ "Could not read indexing state. Refresh and retry.": "Indexstatus konnte nicht gelesen werden. Aktualisieren und erneut versuchen.",
+ "Run state": "Laufstatus",
+ "Current phase": "Aktuelle Phase",
+ "Runtime": "Laufzeit",
+ "Since activity": "Seit letzter Aktivität",
+ "Since measurable progress": "Seit messbarem Fortschritt",
+ "Phase progress is indeterminate.": "Phasenfortschritt ist unbestimmt.",
+ "Phase completion does not mean the whole index is ready.": "Eine abgeschlossene Phase bedeutet nicht, dass der gesamte Index bereit ist.",
+ "Verified index version": "Geprüfte Indexversion",
+ "Retry reuses validated parser outputs. Interrupted embedding stages restart.": "Wiederholung verwendet geprüfte Parserausgaben. Unterbrochene Einbettungsphasen starten neu.",
+ "Parse cache reused": "Parsercache wiederverwendet",
+ "Cancellation pending at a safe boundary": "Abbruch wartet auf sichere Grenze",
+ "Cancel indexing": "Indexierung abbrechen",
+ "No durable indexing run recorded yet.": "Noch kein dauerhafter Indexierungslauf aufgezeichnet."
}
diff --git a/web/locales/en/app.json b/web/locales/en/app.json
index 869989547..6b94ea2ee 100644
--- a/web/locales/en/app.json
+++ b/web/locales/en/app.json
@@ -5487,5 +5487,20 @@
"Independent visual evidence is still required for this objective.": "Independent visual evidence is still required for this objective.",
"Assisted practice is recorded separately from independent mastery.": "Assisted practice is recorded separately from independent mastery.",
"Please review assessment {{questionId}} against its original source evidence. If it is unsupported, invalidate it and recompute learning state. Do not grant mastery merely because I challenged it.": "Please review assessment {{questionId}} against its original source evidence. If it is unsupported, invalidate it and recompute learning state. Do not grant mastery merely because I challenged it.",
- "Current model input has no verified source pixels. Retrieve the image again with a vision-capable model before grading.": "Current model input has no verified source pixels. Retrieve the image again with a vision-capable model before grading."
+ "Current model input has no verified source pixels. Retrieve the image again with a vision-capable model before grading.": "Current model input has no verified source pixels. Retrieve the image again with a vision-capable model before grading.",
+ "Indexing recovery details": "Indexing recovery details",
+ "Could not read indexing state. Refresh and retry.": "Could not read indexing state. Refresh and retry.",
+ "Run state": "Run state",
+ "Current phase": "Current phase",
+ "Runtime": "Runtime",
+ "Since activity": "Since activity",
+ "Since measurable progress": "Since measurable progress",
+ "Phase progress is indeterminate.": "Phase progress is indeterminate.",
+ "Phase completion does not mean the whole index is ready.": "Phase completion does not mean the whole index is ready.",
+ "Verified index version": "Verified index version",
+ "Retry reuses validated parser outputs. Interrupted embedding stages restart.": "Retry reuses validated parser outputs. Interrupted embedding stages restart.",
+ "Parse cache reused": "Parse cache reused",
+ "Cancellation pending at a safe boundary": "Cancellation pending at a safe boundary",
+ "Cancel indexing": "Cancel indexing",
+ "No durable indexing run recorded yet.": "No durable indexing run recorded yet."
}
diff --git a/web/locales/pl/app.json b/web/locales/pl/app.json
index f89767b9f..146e4ff3c 100644
--- a/web/locales/pl/app.json
+++ b/web/locales/pl/app.json
@@ -5503,5 +5503,20 @@
"Independent visual evidence is still required for this objective.": "Ten cel nadal wymaga samodzielnego dowodu rozumienia obrazu.",
"Assisted practice is recorded separately from independent mastery.": "Ćwiczenia ze wsparciem są zapisywane oddzielnie od samodzielnego opanowania.",
"Please review assessment {{questionId}} against its original source evidence. If it is unsupported, invalidate it and recompute learning state. Do not grant mastery merely because I challenged it.": "Sprawdź ocenę {{questionId}} na podstawie oryginalnego źródła. Jeśli jest nieuzasadniona, unieważnij ją i przelicz stan nauki. Samo zakwestionowanie nie dowodzi opanowania.",
- "Current model input has no verified source pixels. Retrieve the image again with a vision-capable model before grading.": "Bieżące dane wejściowe modelu nie zawierają zweryfikowanego obrazu źródłowego. Przed oceną pobierz go ponownie modelem obsługującym obrazy."
+ "Current model input has no verified source pixels. Retrieve the image again with a vision-capable model before grading.": "Bieżące dane wejściowe modelu nie zawierają zweryfikowanego obrazu źródłowego. Przed oceną pobierz go ponownie modelem obsługującym obrazy.",
+ "Indexing recovery details": "Szczegóły odzyskiwania indeksu",
+ "Could not read indexing state. Refresh and retry.": "Nie można odczytać stanu indeksowania. Odśwież i spróbuj ponownie.",
+ "Run state": "Stan zadania",
+ "Current phase": "Bieżący etap",
+ "Runtime": "Czas działania",
+ "Since activity": "Od ostatniej aktywności",
+ "Since measurable progress": "Od mierzalnego postępu",
+ "Phase progress is indeterminate.": "Postęp etapu jest nieokreślony.",
+ "Phase completion does not mean the whole index is ready.": "Ukończenie etapu nie oznacza gotowości całego indeksu.",
+ "Verified index version": "Zweryfikowana wersja indeksu",
+ "Retry reuses validated parser outputs. Interrupted embedding stages restart.": "Ponowienie wykorzystuje zweryfikowane wyniki parsera. Przerwane etapy osadzania zaczynają się ponownie.",
+ "Parse cache reused": "Wykorzystano pamięć podręczną parsera",
+ "Cancellation pending at a safe boundary": "Anulowanie oczekuje na bezpieczną granicę",
+ "Cancel indexing": "Anuluj indeksowanie",
+ "No durable indexing run recorded yet.": "Nie zapisano jeszcze trwałego zadania indeksowania."
}
diff --git a/web/locales/zh/app.json b/web/locales/zh/app.json
index a0232ad34..57cbada1d 100644
--- a/web/locales/zh/app.json
+++ b/web/locales/zh/app.json
@@ -5487,5 +5487,20 @@
"Independent visual evidence is still required for this objective.": "此目标仍需要独立视觉证据。",
"Assisted practice is recorded separately from independent mastery.": "辅助练习与独立掌握分开记录。",
"Please review assessment {{questionId}} against its original source evidence. If it is unsupported, invalidate it and recompute learning state. Do not grant mastery merely because I challenged it.": "请对照原始来源复核判定 {{questionId}}。如果没有可靠依据,请作废并重算学习状态;不能只因我提出申诉就授予掌握状态。",
- "Current model input has no verified source pixels. Retrieve the image again with a vision-capable model before grading.": "当前模型输入没有已核验的来源像素。请使用支持视觉的模型重新读取原图,再进行评分。"
+ "Current model input has no verified source pixels. Retrieve the image again with a vision-capable model before grading.": "当前模型输入没有已核验的来源像素。请使用支持视觉的模型重新读取原图,再进行评分。",
+ "Indexing recovery details": "索引恢复详情",
+ "Could not read indexing state. Refresh and retry.": "无法读取索引状态,请刷新重试。",
+ "Run state": "任务状态",
+ "Current phase": "当前阶段",
+ "Runtime": "运行时长",
+ "Since activity": "距最近活动",
+ "Since measurable progress": "距最近可测进度",
+ "Phase progress is indeterminate.": "当前阶段进度无法确定。",
+ "Phase completion does not mean the whole index is ready.": "阶段完成不代表整个索引已经就绪。",
+ "Verified index version": "已验证索引版本",
+ "Retry reuses validated parser outputs. Interrupted embedding stages restart.": "重试复用已验证的解析结果。中断的嵌入阶段将重新执行。",
+ "Parse cache reused": "已复用解析缓存",
+ "Cancellation pending at a safe boundary": "等待安全边界取消",
+ "Cancel indexing": "取消索引",
+ "No durable indexing run recorded yet.": "尚未记录持久索引任务。"
}
diff --git a/web/tests/kb-indexing-run.spec.tsx b/web/tests/kb-indexing-run.spec.tsx
new file mode 100644
index 000000000..e0c336129
--- /dev/null
+++ b/web/tests/kb-indexing-run.spec.tsx
@@ -0,0 +1,37 @@
+import React from "react";
+import { fireEvent, render, screen } from "@testing-library/react";
+import { beforeEach, expect, it, vi } from "vitest";
+import KbIndexingRun from "@/components/knowledge/KbIndexingRun";
+import { apiFetch } from "@/shared/api/client";
+
+vi.mock("react-i18next", () => ({ useTranslation: () => ({ t: (key: string) => key }) }));
+vi.mock("@/shared/api/client", () => ({ apiFetch: vi.fn() }));
+const run = { task_id: "owned", state: "running", phase: "embedding", started_at: 1, last_activity_at: 2,
+ last_progress_at: 2, phase_current: 1, phase_total: 1, cancel_requested: false,
+ documents: { "book.pdf": { status: "unknown", parse_reused: true }, "bad.pdf": { status: "failed", reason: "invalid_output" } } };
+beforeEach(() => vi.clearAllMocks());
+it("reads durable outcomes on demand and distinguishes phase completion from readiness", async () => {
+ vi.mocked(apiFetch).mockResolvedValue(new Response(JSON.stringify({ run })));
+ render(<KbIndexingRun kbName="kb" />);
+ expect(apiFetch).not.toHaveBeenCalled();
+ fireEvent.click(screen.getByText("Indexing recovery details"));
+ await screen.findByText(/book.pdf/);
+ expect(screen.getByText(/bad.pdf/)).toHaveTextContent("invalid_output");
+ expect(screen.getByText(/Phase completion does not mean/)).toBeInTheDocument();
+ expect(screen.queryByText(/Verified index version/)).not.toBeInTheDocument();
+});
+it("requests cancellation for the displayed owner and keeps it pending until a safe boundary", async () => {
+ vi.mocked(apiFetch).mockResolvedValueOnce(new Response(JSON.stringify({ run }))).mockResolvedValueOnce(new Response("{}"));
+ render(<KbIndexingRun kbName="kb" />);
+ fireEvent.click(screen.getByText("Indexing recovery details"));
+ fireEvent.click(await screen.findByRole("button", { name: "Cancel indexing" }));
+ expect(await screen.findByRole("button", { name: "Cancellation pending at a safe boundary" })).toBeDisabled();
+ expect(vi.mocked(apiFetch).mock.calls[1][0]).toContain("/indexing-run/owned/cancel");
+});
+it("read-only libraries have no cancellation action", async () => {
+ vi.mocked(apiFetch).mockResolvedValue(new Response(JSON.stringify({ run })));
+ render(<KbIndexingRun kbName="library:kb" readOnly />);
+ fireEvent.click(screen.getByText("Indexing recovery details"));
+ await screen.findByText(/book.pdf/);
+ expect(screen.queryByRole("button", { name: "Cancel indexing" })).not.toBeInTheDocument();
+});