Skip to content

DeepTutor: Lifelong Personalized Tutoring. https://deeptutor.info/.

fix(knowledge): surface swallowed KB status, progress and folder-sync failures (#1706)

evan188199-tech50a3ba4e4dcd
diff --git a/deeptutor/api/routers/knowledge.py b/deeptutor/api/routers/knowledge.py
index 5510f9134..ea812e3bb 100644
--- a/deeptutor/api/routers/knowledge.py
+++ b/deeptutor/api/routers/knowledge.py
@@ -3985,8 +3985,16 @@ async def run_reindex_task(
error=error_msg,
**failure_metadata,
)
- except Exception:
- pass
+ except Exception as progress_err:
+ # The task-level error above stays authoritative; degrade loudly
+ # instead of silently, so a stale KB progress view is explainable.
+ logger.warning(
+ "[%s] Re-index of '%s' failed, and persisting its error progress "
+ "also failed (%s); the KB progress view may stay stale.",
+ task_id,
+ kb_name,
+ progress_err,
+ )
task_stream_manager.emit_failed(task_id, error_msg, **failure_metadata)
@@ -4252,6 +4260,23 @@ async def clear_progress(kb_name: str):
raise HTTPException(status_code=500, detail=str(e))
+def _progress_age_seconds(timestamp: object) -> float | None:
+ """Age of a progress snapshot timestamp in seconds, ``None`` if unknown.
+
+ A malformed timestamp means freshness cannot be determined; callers fall
+ back to their inactive/stale handling, and the anomaly is logged so a
+ corrupt snapshot cannot silently disable liveness detection.
+ """
+ if not timestamp:
+ return None
+ try:
+ parsed = datetime.fromisoformat(str(timestamp))
+ return (datetime.now() - parsed).total_seconds()
+ except (TypeError, ValueError) as exc:
+ logger.warning("Cannot determine progress age from timestamp %r: %s", timestamp, exc)
+ return None
+
+
@ws_router.websocket("/knowledge-bases/{kb_name}/progress")
async def websocket_progress(websocket: WebSocket, kb_name: str):
"""WebSocket endpoint for real-time progress updates"""
@@ -4378,16 +4403,17 @@ async def websocket_progress(websocket: WebSocket, kb_name: str):
# Fast path: no active task — send current state and close immediately
# This prevents infinite polling loops for ready or legacy KBs.
has_active_task = False
+ initial_ts: object = None
+ initial_age_seconds: float | None = None
if initial_progress:
stage = initial_progress.get("stage")
if stage not in ("completed", "error", None):
ts = initial_progress.get("timestamp")
if ts:
- try:
- age = (datetime.now() - datetime.fromisoformat(ts)).total_seconds()
- has_active_task = age < 120
- except Exception:
- pass
+ initial_ts = ts
+ initial_age_seconds = _progress_age_seconds(ts)
+ if initial_age_seconds is not None:
+ has_active_task = initial_age_seconds < 120
if not has_active_task and not expected_task_id:
if kb_is_ready:
@@ -4429,14 +4455,16 @@ async def websocket_progress(websocket: WebSocket, kb_name: str):
elif stage == "error" or not kb_is_ready:
should_send = True
elif stage != "completed" and timestamp:
- try:
- progress_time = datetime.fromisoformat(timestamp)
- now = datetime.now()
- age_seconds = (now - progress_time).total_seconds()
- if age_seconds < 300:
- should_send = True
- except Exception:
- pass
+ # The same snapshot timestamp may already have been assessed
+ # above; reuse that result so one corrupt timestamp yields one
+ # warning instead of one per freshness check.
+ age_seconds = (
+ initial_age_seconds
+ if timestamp is initial_ts
+ else _progress_age_seconds(timestamp)
+ )
+ if age_seconds is not None and age_seconds < 300:
+ should_send = True
if should_send:
await websocket.send_json({"type": "progress", "data": initial_progress})
@@ -4513,19 +4541,24 @@ async def websocket_progress(websocket: WebSocket, kb_name: str):
logger.debug(f"Progress WS error: {e}")
try:
await websocket.send_json({"type": "error", "message": str(e)})
- except Exception:
- pass
+ except Exception as send_error:
+ # Usually a client that is already gone; trace it so a genuinely
+ # broken send path cannot vanish.
+ logger.debug("Failed to deliver error frame on progress WS: %s", send_error)
finally:
await broadcaster.disconnect(subscription_key, websocket)
try:
await websocket.close()
- except Exception:
- pass
+ except Exception as close_error:
+ # Closing an already-disconnected socket is expected; keep a trace
+ # for anything else.
+ logger.debug("Failed to close progress WS: %s", close_error)
if user_token is not None:
try:
reset_current_user(user_token)
- except Exception:
- pass
+ except Exception as reset_error:
+ # Leaking the user context is a real fault, not a disconnect.
+ logger.warning("Failed to reset user context after progress WS: %s", reset_error)
@router.post("/knowledge-bases/{kb_name}/link-folder", response_model=LinkedFolderInfo)
diff --git a/deeptutor/knowledge/manager.py b/deeptutor/knowledge/manager.py
index 677512b64..bb5fab91e 100644
--- a/deeptutor/knowledge/manager.py
+++ b/deeptutor/knowledge/manager.py
@@ -612,10 +612,19 @@ class KnowledgeBaseManager:
kb_dir = self.base_dir / name
if kb_dir.is_dir():
kb_config["index_versions"] = inspect_kb_versions(kb_dir, provider)
- except Exception: # pragma: no cover - best-effort metadata
- pass
+ except Exception as exc: # best-effort metadata
+ logger.warning(
+ f"Failed to refresh embedding/index metadata for KB '{name}' "
+ f"on ready transition: {exc}"
+ )
- self._save_config()
+ try:
+ self._save_config()
+ except Exception as exc:
+ logger.error(
+ f"Failed to persist KB status update for '{name}' (status='{status}'): {exc}"
+ )
+ raise
self._sync_kb_to_pb(name, kb_config)
def get_kb_entry(self, name: str) -> dict | None:
@@ -2212,8 +2221,14 @@ class KnowledgeBaseManager:
if p.exists():
mtime = datetime.fromtimestamp(p.stat().st_mtime)
file_states[file_path] = mtime.isoformat()
- except Exception:
- pass
+ except Exception as exc:
+ # Recording failed: leave the file un-recorded so the
+ # next scan re-syncs it instead of trusting a missing
+ # mtime, and make the degradation visible in logs.
+ logger.warning(
+ f"Failed to record sync state for '{file_path}' in folder "
+ f"'{folder_id}' of KB '{kb_name}': {exc}"
+ )
folder["synced_files"] = file_states
folder["file_count"] = len(file_states)
diff --git a/tests/api/test_knowledge_progress_ws.py b/tests/api/test_knowledge_progress_ws.py
index eceab7842..3ddb61556 100644
--- a/tests/api/test_knowledge_progress_ws.py
+++ b/tests/api/test_knowledge_progress_ws.py
@@ -1,6 +1,8 @@
from __future__ import annotations
+import asyncio
import json
+import logging
from pathlib import Path
from fastapi import FastAPI
@@ -76,3 +78,196 @@ def test_orphaned_live_progress_becomes_retryable_error(monkeypatch, tmp_path: P
persisted = json.loads((base_dir / "kb" / ".progress.json").read_text(encoding="utf-8"))
assert persisted["stage"] == "error"
assert persisted["task_id"] == "orphaned-after-restart"
+
+
+def test_malformed_progress_timestamp_is_logged_and_degrades(
+ monkeypatch, tmp_path: Path, caplog
+) -> None:
+ """An unparseable freshness timestamp falls back safely AND stays visible."""
+ base_dir = tmp_path / "knowledge_bases"
+ _write_progress(
+ base_dir,
+ {
+ "task_id": "stuck-task",
+ "stage": "processing_documents",
+ "message": "working",
+ "progress_percent": 40,
+ "timestamp": "not-an-iso-timestamp",
+ },
+ )
+
+ with caplog.at_level(logging.WARNING, logger="deeptutor.api.routers.knowledge"):
+ with _client(monkeypatch, base_dir).websocket_connect(
+ "/ws/knowledge-bases/kb/progress"
+ ) as websocket:
+ frame = websocket.receive_json()
+
+ # Degradation: the snapshot is still replayed on the no-task fast path
+ # instead of blocking the endpoint.
+ assert frame["type"] == "progress"
+ assert frame["data"]["stage"] == "processing_documents"
+ # Visibility: the parse failure must reach the logs.
+ assert any("not-an-iso-timestamp" in record.getMessage() for record in caplog.records), (
+ f"expected a log mentioning the malformed timestamp, got: {caplog.records}"
+ )
+
+
+def test_malformed_timestamp_with_task_id_warns_once(monkeypatch, tmp_path: Path, caplog) -> None:
+ """A corrupt snapshot timestamp is reported once even when both freshness
+ checks (active-task detection and replay decision) consult it."""
+ base_dir = tmp_path / "knowledge_bases"
+ _write_progress(
+ base_dir,
+ {
+ # Snapshot predates task ids: the freshness checks still apply,
+ # but the expected task id cannot short-circuit them.
+ "stage": "processing_documents",
+ "message": "working",
+ "progress_percent": 40,
+ "timestamp": "not-an-iso-timestamp",
+ },
+ )
+
+ class _ReadyKBManager:
+ def __init__(self, *_args, **_kwargs) -> None:
+ pass
+
+ def get_info(self, _name: str) -> dict:
+ return {"statistics": {"rag_initialized": True}}
+
+ class _LiveTaskManager:
+ @staticmethod
+ def get_instance() -> "_LiveTaskManager":
+ return _LiveTaskManager()
+
+ def get_task_metadata(self, _task_id: str) -> dict:
+ return {"status": "processing"}
+
+ class _TasklessSocket:
+ query_params: dict[str, str] = {"task_id": "live-task"}
+
+ async def accept(self) -> None:
+ return None
+
+ async def send_json(self, payload: dict) -> None:
+ return None
+
+ async def receive_text(self) -> str:
+ raise RuntimeError("client went away")
+
+ async def close(self) -> None:
+ return None
+
+ async def allow(_websocket):
+ return None
+
+ async def _noop(_key, _websocket):
+ return None
+
+ from deeptutor.api.utils.progress_broadcaster import ProgressBroadcaster
+
+ broadcaster = ProgressBroadcaster.get_instance()
+ monkeypatch.setattr(auth, "ws_require_auth", allow)
+ monkeypatch.setattr(knowledge_router, "_current_kb_base_dir", lambda: base_dir)
+ monkeypatch.setattr(knowledge_router, "KnowledgeBaseManager", _ReadyKBManager)
+ monkeypatch.setattr(knowledge_router, "TaskIDManager", _LiveTaskManager)
+ monkeypatch.setattr(broadcaster, "connect", _noop)
+ monkeypatch.setattr(broadcaster, "disconnect", _noop)
+
+ with caplog.at_level(logging.WARNING, logger="deeptutor.api.routers.knowledge"):
+ asyncio.run(knowledge_router.websocket_progress(_TasklessSocket(), "kb"))
+
+ warnings = [
+ record.getMessage()
+ for record in caplog.records
+ if "not-an-iso-timestamp" in record.getMessage()
+ ]
+ assert len(warnings) == 1, (
+ f"expected exactly one warning for the corrupt timestamp, got: {warnings}"
+ )
+
+
+def test_cleanup_and_error_delivery_failures_are_logged_not_silent(
+ monkeypatch, tmp_path: Path, caplog
+) -> None:
+ """Error-frame send, close, and user-context reset failures become log lines."""
+ from deeptutor.api.utils.progress_broadcaster import ProgressBroadcaster
+ from deeptutor.multi_user import context as multi_user_context
+
+ class _BrokenPipeWebSocket:
+ query_params: dict[str, str] = {}
+
+ async def accept(self) -> None:
+ return None
+
+ async def send_json(self, payload: dict) -> None:
+ raise RuntimeError("send pipe broken")
+
+ async def close(self) -> None:
+ raise RuntimeError("socket already closed")
+
+ class _ExplodingTracker:
+ def __init__(self, kb_name: str, base_dir: Path):
+ pass
+
+ def get_progress(self) -> dict:
+ raise ValueError("progress snapshot read failed")
+
+ async def allow(_websocket):
+ return "opaque-token"
+
+ async def _noop(_key, _websocket):
+ return None
+
+ broadcaster = ProgressBroadcaster.get_instance()
+ monkeypatch.setattr(auth, "ws_require_auth", allow)
+ monkeypatch.setattr(knowledge_router, "_current_kb_base_dir", lambda: tmp_path)
+ monkeypatch.setattr(knowledge_router, "ProgressTracker", _ExplodingTracker)
+ monkeypatch.setattr(broadcaster, "connect", _noop)
+ monkeypatch.setattr(broadcaster, "disconnect", _noop)
+
+ def _reset_boom(_token):
+ raise RuntimeError("context reset failed")
+
+ monkeypatch.setattr(multi_user_context, "reset_current_user", _reset_boom)
+
+ with caplog.at_level(logging.DEBUG, logger="deeptutor.api.routers.knowledge"):
+ asyncio.run(knowledge_router.websocket_progress(_BrokenPipeWebSocket(), "kb"))
+
+ messages = [record.getMessage() for record in caplog.records]
+ # The original failure keeps its existing debug trace.
+ assert any("progress snapshot read failed" in message for message in messages)
+ # A failed error-frame delivery (usually a gone client) is now traced too.
+ assert any("send pipe broken" in message for message in messages), messages
+ # A failed close no longer disappears.
+ assert any("socket already closed" in message for message in messages), messages
+ # A failed user-context reset is a real fault: it must warn, not vanish.
+ warnings = [
+ record.getMessage() for record in caplog.records if record.levelno >= logging.WARNING
+ ]
+ assert any("context reset failed" in message for message in warnings), messages
+
+
+def test_client_disconnect_is_not_reported_as_error(monkeypatch, tmp_path: Path, caplog) -> None:
+ """Teardown after a normal exchange stays quiet at warning level and above."""
+ base_dir = tmp_path / "knowledge_bases"
+ _write_progress(
+ base_dir,
+ {
+ "task_id": "completed-task",
+ "stage": "completed",
+ "message": "done",
+ "progress_percent": 100,
+ "timestamp": "2026-09-02T00:00:00",
+ },
+ )
+
+ with caplog.at_level(logging.WARNING, logger="deeptutor.api.routers.knowledge"):
+ with _client(monkeypatch, base_dir).websocket_connect(
+ "/ws/knowledge-bases/kb/progress?task_id=completed-task"
+ ) as websocket:
+ frame = websocket.receive_json()
+
+ assert frame["data"]["stage"] == "completed"
+ noisy = [record.getMessage() for record in caplog.records]
+ assert not noisy, f"disconnect path produced warning-level logs: {noisy}"
diff --git a/tests/api/test_knowledge_router.py b/tests/api/test_knowledge_router.py
index f46cc58df..eea826b95 100644
--- a/tests/api/test_knowledge_router.py
+++ b/tests/api/test_knowledge_router.py
@@ -4,6 +4,7 @@ import asyncio
import hashlib
import importlib
import json
+import logging
from pathlib import Path
from types import SimpleNamespace
@@ -1920,6 +1921,72 @@ def test_reindex_task_preserves_prepublication_failure(monkeypatch, tmp_path: Pa
assert task["error"] == "original indexing failure"
+def test_reindex_task_surfaces_failed_error_progress_write(
+ monkeypatch, tmp_path: Path, caplog: pytest.LogCaptureFixture
+) -> None:
+ """A failing error-progress write must not vanish silently.
+
+ The rebuild's task-level failure stays authoritative; the router logs the
+ degraded progress write instead of swallowing it with ``except: pass``.
+ """
+ base_dir = tmp_path / "knowledge_bases"
+ raw_dir = base_dir / "kb" / "raw"
+ raw_dir.mkdir(parents=True)
+ (raw_dir / "fixture.txt").write_text("fixture", encoding="utf-8")
+ (base_dir / "kb_config.json").write_text(
+ json.dumps(
+ {
+ "knowledge_bases": {
+ "kb": {
+ "path": "kb",
+ "rag_provider": "lightrag",
+ "status": "processing",
+ }
+ }
+ }
+ ),
+ encoding="utf-8",
+ )
+
+ class _FailingRagService:
+ def __init__(self, *_args, **_kwargs) -> None:
+ pass
+
+ async def initialize(self, *_args, **_kwargs) -> bool:
+ raise RuntimeError("original indexing failure")
+
+ rag_service_module = importlib.import_module("deeptutor.services.rag.service")
+ monkeypatch.setattr(rag_service_module, "RAGService", _FailingRagService)
+
+ class _ErrorWriteFailsTracker(knowledge_router_module.ProgressTracker):
+ def update(self, stage, *args, **kwargs):
+ if stage is knowledge_router_module.ProgressStage.ERROR:
+ raise OSError("synthetic error progress write failure")
+ return super().update(stage, *args, **kwargs)
+
+ monkeypatch.setattr(knowledge_router_module, "ProgressTracker", _ErrorWriteFailsTracker)
+
+ task_id = knowledge_router_module._build_unique_task_id("kb_reindex", "error-progress")
+ with caplog.at_level(logging.WARNING, logger="deeptutor.api.routers.knowledge"):
+ asyncio.run(
+ knowledge_router_module.run_reindex_task(
+ kb_name="kb",
+ base_dir=str(base_dir),
+ task_id=task_id,
+ signature_hash="lightrag",
+ )
+ )
+
+ task = knowledge_router_module.TaskIDManager.get_instance().get_task_metadata(task_id)
+ assert task is not None
+ assert task["status"] == "error"
+ assert task["error"] == "original indexing failure"
+ warnings = [
+ record.getMessage() for record in caplog.records if record.levelno >= logging.WARNING
+ ]
+ assert any("error progress" in message and "kb" in message for message in warnings), warnings
+
+
@pytest.mark.parametrize("failed_sink", ["progress_file", "central_config"])
def test_reindex_task_does_not_report_a_published_version_as_failed_when_bookkeeping_fails(
monkeypatch, tmp_path: Path, failed_sink: str
diff --git a/tests/knowledge/test_linked_folder_sync.py b/tests/knowledge/test_linked_folder_sync.py
index 14221b004..327c8dc63 100644
--- a/tests/knowledge/test_linked_folder_sync.py
+++ b/tests/knowledge/test_linked_folder_sync.py
@@ -10,9 +10,11 @@ The atomicity contract of the shared writer lives in
from __future__ import annotations
import json
+import logging
from pathlib import Path
from deeptutor.api.routers.knowledge import LinkedFolderInfo
+from deeptutor.knowledge import manager as manager_module
from deeptutor.knowledge.manager import KnowledgeBaseManager
@@ -88,6 +90,54 @@ def test_empty_successful_sync_records_last_sync_without_changing_file_state(
assert after["synced_files"] == before["synced_files"]
+def test_update_folder_sync_state_mtime_failure_is_visible_not_fake_synced(
+ tmp_path: Path,
+ caplog,
+ monkeypatch,
+) -> None:
+ """A per-file mtime recording failure must surface, not vanish.
+
+ ``update_folder_sync_state()`` used to swallow stat/fromtimestamp errors
+ with ``except Exception: pass``. The sync then reported success while the
+ file's state was never recorded, hiding the degradation. The failure must
+ be logged and the file must stay un-recorded so the next scan re-syncs it.
+ """
+ manager, metadata_file, folder_id, doc = _manager_with_linked_folder(tmp_path)
+
+ flavour = type(Path()) # PosixPath on POSIX, WindowsPath on Windows
+
+ class _VanishedMidStatPath(flavour):
+ """exists() wins the race, stat() loses it (source vanished mid-sync)."""
+
+ def exists(self, **kwargs):
+ return True
+
+ def stat(self, **kwargs):
+ raise OSError("source vanished before stat")
+
+ monkeypatch.setattr(manager_module, "Path", _VanishedMidStatPath)
+
+ with caplog.at_level(logging.WARNING, logger="deeptutor.knowledge.manager"):
+ manager.update_folder_sync_state("kb", folder_id, [str(doc)])
+
+ # Visibility: the swallowed failure is now a logged warning naming the file.
+ warnings_for_doc = [
+ record
+ for record in caplog.records
+ if record.levelno >= logging.WARNING and str(doc) in record.getMessage()
+ ]
+ assert warnings_for_doc, f"expected a logged warning mentioning {doc}, got none"
+
+ # Fallback semantics: the file is not marked synced, so the next scan
+ # re-detects it instead of trusting a fabricated "already synced" state.
+ on_disk = json.loads(metadata_file.read_text(encoding="utf-8"))
+ folder = on_disk["linked_folders"][0]
+ assert folder["last_sync"]
+ assert str(doc) not in folder["synced_files"]
+ assert folder["file_count"] == 0
+ assert manager.detect_folder_changes("kb", folder_id)["new_files"] == [str(doc)]
+
+
def test_unlink_removes_only_the_source_registration(tmp_path: Path) -> None:
manager, _metadata_file, folder_id, _doc = _manager_with_linked_folder(tmp_path)
kb_dir = manager.base_dir / "kb"
diff --git a/tests/knowledge/test_manager_update_kb_status_failures.py b/tests/knowledge/test_manager_update_kb_status_failures.py
new file mode 100644
index 000000000..74a6fc6d5
--- /dev/null
+++ b/tests/knowledge/test_manager_update_kb_status_failures.py
@@ -0,0 +1,72 @@
+"""Failure-visibility tests for ``KnowledgeBaseManager.update_kb_status``.
+
+A bare ``except Exception: pass`` around the ready-transition metadata
+refresh meant that when refreshing embedding signatures or
+``index_versions`` failed, the failure left no trace and the persisted KB
+entry silently diverged from the on-disk index state that the UI renders.
+
+These tests pin the contract that (a) a metadata refresh failure is logged
+but stays best-effort, (b) a config-persist failure is logged at error level
+and still reaches the caller as an exception, and (c) a successful update
+persists the new status.
+"""
+
+from __future__ import annotations
+
+import logging
+from pathlib import Path
+
+import pytest
+
+from deeptutor.knowledge.manager import KnowledgeBaseManager
+
+
+def test_successful_update_persists_status(tmp_path: Path) -> None:
+ manager = KnowledgeBaseManager(base_dir=str(tmp_path))
+
+ manager.update_kb_status(name="kb1", status="processing")
+
+ assert manager.get_kb_status("kb1")["status"] == "processing"
+
+
+def test_persist_failure_is_logged_and_raised(
+ tmp_path: Path, monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture
+) -> None:
+ manager = KnowledgeBaseManager(base_dir=str(tmp_path))
+
+ def _fail_save() -> None:
+ raise OSError("disk full")
+
+ monkeypatch.setattr(manager, "_save_config", _fail_save)
+
+ with caplog.at_level(logging.ERROR, logger="deeptutor.knowledge.manager"):
+ with pytest.raises(OSError, match="disk full"):
+ manager.update_kb_status(name="kb1", status="error")
+
+ assert any(
+ "kb1" in record.message and record.levelno == logging.ERROR for record in caplog.records
+ )
+
+
+def test_ready_metadata_failure_is_logged(
+ tmp_path: Path, monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture
+) -> None:
+ from deeptutor.knowledge import manager as manager_module
+
+ manager = KnowledgeBaseManager(base_dir=str(tmp_path))
+ (tmp_path / "kb1").mkdir()
+ (tmp_path / "kb1" / "version-1").mkdir()
+
+ def _boom(kb_dir: Path, provider: str) -> list[dict]:
+ raise RuntimeError("docstore exploded")
+
+ monkeypatch.setattr(manager_module, "inspect_kb_versions", _boom)
+
+ with caplog.at_level(logging.WARNING, logger="deeptutor.knowledge.manager"):
+ manager.update_kb_status(name="kb1", status="ready")
+
+ # The metadata refresh is best-effort: the status write itself still lands.
+ assert manager.get_kb_status("kb1")["status"] == "ready"
+ assert any(
+ "kb1" in record.message and record.levelno == logging.WARNING for record in caplog.records
+ )