DeepTutor: Lifelong Personalized Tutoring. https://deeptutor.info/.
Python ai-agentsai-tutorclawbotcli-tooldeepresearchinteractive-learninglearningmulti-agent-systemsrag
fix(knowledge): surface swallowed KB status, progress and folder-sync failures (#1706)
evan188199-tech50a3ba4e4dcd
diff --git a/deeptutor/api/routers/knowledge.py b/deeptutor/api/routers/knowledge.pyindex 5510f9134..ea812e3bb 100644--- a/deeptutor/api/routers/knowledge.py+++ b/deeptutor/api/routers/knowledge.py@@ -3985,8 +3985,16 @@ async def run_reindex_task(error=error_msg,**failure_metadata,)- except Exception:- pass+ except Exception as progress_err:+ # The task-level error above stays authoritative; degrade loudly+ # instead of silently, so a stale KB progress view is explainable.+ logger.warning(+ "[%s] Re-index of '%s' failed, and persisting its error progress "+ "also failed (%s); the KB progress view may stay stale.",+ task_id,+ kb_name,+ progress_err,+ )task_stream_manager.emit_failed(task_id, error_msg, **failure_metadata)@@ -4252,6 +4260,23 @@ async def clear_progress(kb_name: str):raise HTTPException(status_code=500, detail=str(e))+def _progress_age_seconds(timestamp: object) -> float | None:+ """Age of a progress snapshot timestamp in seconds, ``None`` if unknown.++ A malformed timestamp means freshness cannot be determined; callers fall+ back to their inactive/stale handling, and the anomaly is logged so a+ corrupt snapshot cannot silently disable liveness detection.+ """+ if not timestamp:+ return None+ try:+ parsed = datetime.fromisoformat(str(timestamp))+ return (datetime.now() - parsed).total_seconds()+ except (TypeError, ValueError) as exc:+ logger.warning("Cannot determine progress age from timestamp %r: %s", timestamp, exc)+ return None++@ws_router.websocket("/knowledge-bases/{kb_name}/progress")async def websocket_progress(websocket: WebSocket, kb_name: str):"""WebSocket endpoint for real-time progress updates"""@@ -4378,16 +4403,17 @@ async def websocket_progress(websocket: WebSocket, kb_name: str):# Fast path: no active task — send current state and close immediately# This prevents infinite polling loops for ready or legacy KBs.has_active_task = False+ initial_ts: object = None+ initial_age_seconds: float | None = Noneif initial_progress:stage = initial_progress.get("stage")if stage not in ("completed", "error", None):ts = initial_progress.get("timestamp")if ts:- try:- age = (datetime.now() - datetime.fromisoformat(ts)).total_seconds()- has_active_task = age < 120- except Exception:- pass+ initial_ts = ts+ initial_age_seconds = _progress_age_seconds(ts)+ if initial_age_seconds is not None:+ has_active_task = initial_age_seconds < 120if not has_active_task and not expected_task_id:if kb_is_ready:@@ -4429,14 +4455,16 @@ async def websocket_progress(websocket: WebSocket, kb_name: str):elif stage == "error" or not kb_is_ready:should_send = Trueelif stage != "completed" and timestamp:- try:- progress_time = datetime.fromisoformat(timestamp)- now = datetime.now()- age_seconds = (now - progress_time).total_seconds()- if age_seconds < 300:- should_send = True- except Exception:- pass+ # The same snapshot timestamp may already have been assessed+ # above; reuse that result so one corrupt timestamp yields one+ # warning instead of one per freshness check.+ age_seconds = (+ initial_age_seconds+ if timestamp is initial_ts+ else _progress_age_seconds(timestamp)+ )+ if age_seconds is not None and age_seconds < 300:+ should_send = Trueif should_send:await websocket.send_json({"type": "progress", "data": initial_progress})@@ -4513,19 +4541,24 @@ async def websocket_progress(websocket: WebSocket, kb_name: str):logger.debug(f"Progress WS error: {e}")try:await websocket.send_json({"type": "error", "message": str(e)})- except Exception:- pass+ except Exception as send_error:+ # Usually a client that is already gone; trace it so a genuinely+ # broken send path cannot vanish.+ logger.debug("Failed to deliver error frame on progress WS: %s", send_error)finally:await broadcaster.disconnect(subscription_key, websocket)try:await websocket.close()- except Exception:- pass+ except Exception as close_error:+ # Closing an already-disconnected socket is expected; keep a trace+ # for anything else.+ logger.debug("Failed to close progress WS: %s", close_error)if user_token is not None:try:reset_current_user(user_token)- except Exception:- pass+ except Exception as reset_error:+ # Leaking the user context is a real fault, not a disconnect.+ logger.warning("Failed to reset user context after progress WS: %s", reset_error)@router.post("/knowledge-bases/{kb_name}/link-folder", response_model=LinkedFolderInfo)diff --git a/deeptutor/knowledge/manager.py b/deeptutor/knowledge/manager.pyindex 677512b64..bb5fab91e 100644--- a/deeptutor/knowledge/manager.py+++ b/deeptutor/knowledge/manager.py@@ -612,10 +612,19 @@ class KnowledgeBaseManager:kb_dir = self.base_dir / nameif kb_dir.is_dir():kb_config["index_versions"] = inspect_kb_versions(kb_dir, provider)- except Exception: # pragma: no cover - best-effort metadata- pass+ except Exception as exc: # best-effort metadata+ logger.warning(+ f"Failed to refresh embedding/index metadata for KB '{name}' "+ f"on ready transition: {exc}"+ )- self._save_config()+ try:+ self._save_config()+ except Exception as exc:+ logger.error(+ f"Failed to persist KB status update for '{name}' (status='{status}'): {exc}"+ )+ raiseself._sync_kb_to_pb(name, kb_config)def get_kb_entry(self, name: str) -> dict | None:@@ -2212,8 +2221,14 @@ class KnowledgeBaseManager:if p.exists():mtime = datetime.fromtimestamp(p.stat().st_mtime)file_states[file_path] = mtime.isoformat()- except Exception:- pass+ except Exception as exc:+ # Recording failed: leave the file un-recorded so the+ # next scan re-syncs it instead of trusting a missing+ # mtime, and make the degradation visible in logs.+ logger.warning(+ f"Failed to record sync state for '{file_path}' in folder "+ f"'{folder_id}' of KB '{kb_name}': {exc}"+ )folder["synced_files"] = file_statesfolder["file_count"] = len(file_states)diff --git a/tests/api/test_knowledge_progress_ws.py b/tests/api/test_knowledge_progress_ws.pyindex eceab7842..3ddb61556 100644--- a/tests/api/test_knowledge_progress_ws.py+++ b/tests/api/test_knowledge_progress_ws.py@@ -1,6 +1,8 @@from __future__ import annotations+import asyncioimport json+import loggingfrom pathlib import Pathfrom fastapi import FastAPI@@ -76,3 +78,196 @@ def test_orphaned_live_progress_becomes_retryable_error(monkeypatch, tmp_path: Ppersisted = json.loads((base_dir / "kb" / ".progress.json").read_text(encoding="utf-8"))assert persisted["stage"] == "error"assert persisted["task_id"] == "orphaned-after-restart"+++def test_malformed_progress_timestamp_is_logged_and_degrades(+ monkeypatch, tmp_path: Path, caplog+) -> None:+ """An unparseable freshness timestamp falls back safely AND stays visible."""+ base_dir = tmp_path / "knowledge_bases"+ _write_progress(+ base_dir,+ {+ "task_id": "stuck-task",+ "stage": "processing_documents",+ "message": "working",+ "progress_percent": 40,+ "timestamp": "not-an-iso-timestamp",+ },+ )++ with caplog.at_level(logging.WARNING, logger="deeptutor.api.routers.knowledge"):+ with _client(monkeypatch, base_dir).websocket_connect(+ "/ws/knowledge-bases/kb/progress"+ ) as websocket:+ frame = websocket.receive_json()++ # Degradation: the snapshot is still replayed on the no-task fast path+ # instead of blocking the endpoint.+ assert frame["type"] == "progress"+ assert frame["data"]["stage"] == "processing_documents"+ # Visibility: the parse failure must reach the logs.+ assert any("not-an-iso-timestamp" in record.getMessage() for record in caplog.records), (+ f"expected a log mentioning the malformed timestamp, got: {caplog.records}"+ )+++def test_malformed_timestamp_with_task_id_warns_once(monkeypatch, tmp_path: Path, caplog) -> None:+ """A corrupt snapshot timestamp is reported once even when both freshness+ checks (active-task detection and replay decision) consult it."""+ base_dir = tmp_path / "knowledge_bases"+ _write_progress(+ base_dir,+ {+ # Snapshot predates task ids: the freshness checks still apply,+ # but the expected task id cannot short-circuit them.+ "stage": "processing_documents",+ "message": "working",+ "progress_percent": 40,+ "timestamp": "not-an-iso-timestamp",+ },+ )++ class _ReadyKBManager:+ def __init__(self, *_args, **_kwargs) -> None:+ pass++ def get_info(self, _name: str) -> dict:+ return {"statistics": {"rag_initialized": True}}++ class _LiveTaskManager:+ @staticmethod+ def get_instance() -> "_LiveTaskManager":+ return _LiveTaskManager()++ def get_task_metadata(self, _task_id: str) -> dict:+ return {"status": "processing"}++ class _TasklessSocket:+ query_params: dict[str, str] = {"task_id": "live-task"}++ async def accept(self) -> None:+ return None++ async def send_json(self, payload: dict) -> None:+ return None++ async def receive_text(self) -> str:+ raise RuntimeError("client went away")++ async def close(self) -> None:+ return None++ async def allow(_websocket):+ return None++ async def _noop(_key, _websocket):+ return None++ from deeptutor.api.utils.progress_broadcaster import ProgressBroadcaster++ broadcaster = ProgressBroadcaster.get_instance()+ monkeypatch.setattr(auth, "ws_require_auth", allow)+ monkeypatch.setattr(knowledge_router, "_current_kb_base_dir", lambda: base_dir)+ monkeypatch.setattr(knowledge_router, "KnowledgeBaseManager", _ReadyKBManager)+ monkeypatch.setattr(knowledge_router, "TaskIDManager", _LiveTaskManager)+ monkeypatch.setattr(broadcaster, "connect", _noop)+ monkeypatch.setattr(broadcaster, "disconnect", _noop)++ with caplog.at_level(logging.WARNING, logger="deeptutor.api.routers.knowledge"):+ asyncio.run(knowledge_router.websocket_progress(_TasklessSocket(), "kb"))++ warnings = [+ record.getMessage()+ for record in caplog.records+ if "not-an-iso-timestamp" in record.getMessage()+ ]+ assert len(warnings) == 1, (+ f"expected exactly one warning for the corrupt timestamp, got: {warnings}"+ )+++def test_cleanup_and_error_delivery_failures_are_logged_not_silent(+ monkeypatch, tmp_path: Path, caplog+) -> None:+ """Error-frame send, close, and user-context reset failures become log lines."""+ from deeptutor.api.utils.progress_broadcaster import ProgressBroadcaster+ from deeptutor.multi_user import context as multi_user_context++ class _BrokenPipeWebSocket:+ query_params: dict[str, str] = {}++ async def accept(self) -> None:+ return None++ async def send_json(self, payload: dict) -> None:+ raise RuntimeError("send pipe broken")++ async def close(self) -> None:+ raise RuntimeError("socket already closed")++ class _ExplodingTracker:+ def __init__(self, kb_name: str, base_dir: Path):+ pass++ def get_progress(self) -> dict:+ raise ValueError("progress snapshot read failed")++ async def allow(_websocket):+ return "opaque-token"++ async def _noop(_key, _websocket):+ return None++ broadcaster = ProgressBroadcaster.get_instance()+ monkeypatch.setattr(auth, "ws_require_auth", allow)+ monkeypatch.setattr(knowledge_router, "_current_kb_base_dir", lambda: tmp_path)+ monkeypatch.setattr(knowledge_router, "ProgressTracker", _ExplodingTracker)+ monkeypatch.setattr(broadcaster, "connect", _noop)+ monkeypatch.setattr(broadcaster, "disconnect", _noop)++ def _reset_boom(_token):+ raise RuntimeError("context reset failed")++ monkeypatch.setattr(multi_user_context, "reset_current_user", _reset_boom)++ with caplog.at_level(logging.DEBUG, logger="deeptutor.api.routers.knowledge"):+ asyncio.run(knowledge_router.websocket_progress(_BrokenPipeWebSocket(), "kb"))++ messages = [record.getMessage() for record in caplog.records]+ # The original failure keeps its existing debug trace.+ assert any("progress snapshot read failed" in message for message in messages)+ # A failed error-frame delivery (usually a gone client) is now traced too.+ assert any("send pipe broken" in message for message in messages), messages+ # A failed close no longer disappears.+ assert any("socket already closed" in message for message in messages), messages+ # A failed user-context reset is a real fault: it must warn, not vanish.+ warnings = [+ record.getMessage() for record in caplog.records if record.levelno >= logging.WARNING+ ]+ assert any("context reset failed" in message for message in warnings), messages+++def test_client_disconnect_is_not_reported_as_error(monkeypatch, tmp_path: Path, caplog) -> None:+ """Teardown after a normal exchange stays quiet at warning level and above."""+ base_dir = tmp_path / "knowledge_bases"+ _write_progress(+ base_dir,+ {+ "task_id": "completed-task",+ "stage": "completed",+ "message": "done",+ "progress_percent": 100,+ "timestamp": "2026-09-02T00:00:00",+ },+ )++ with caplog.at_level(logging.WARNING, logger="deeptutor.api.routers.knowledge"):+ with _client(monkeypatch, base_dir).websocket_connect(+ "/ws/knowledge-bases/kb/progress?task_id=completed-task"+ ) as websocket:+ frame = websocket.receive_json()++ assert frame["data"]["stage"] == "completed"+ noisy = [record.getMessage() for record in caplog.records]+ assert not noisy, f"disconnect path produced warning-level logs: {noisy}"diff --git a/tests/api/test_knowledge_router.py b/tests/api/test_knowledge_router.pyindex f46cc58df..eea826b95 100644--- a/tests/api/test_knowledge_router.py+++ b/tests/api/test_knowledge_router.py@@ -4,6 +4,7 @@ import asyncioimport hashlibimport importlibimport json+import loggingfrom pathlib import Pathfrom types import SimpleNamespace@@ -1920,6 +1921,72 @@ def test_reindex_task_preserves_prepublication_failure(monkeypatch, tmp_path: Paassert task["error"] == "original indexing failure"+def test_reindex_task_surfaces_failed_error_progress_write(+ monkeypatch, tmp_path: Path, caplog: pytest.LogCaptureFixture+) -> None:+ """A failing error-progress write must not vanish silently.++ The rebuild's task-level failure stays authoritative; the router logs the+ degraded progress write instead of swallowing it with ``except: pass``.+ """+ base_dir = tmp_path / "knowledge_bases"+ raw_dir = base_dir / "kb" / "raw"+ raw_dir.mkdir(parents=True)+ (raw_dir / "fixture.txt").write_text("fixture", encoding="utf-8")+ (base_dir / "kb_config.json").write_text(+ json.dumps(+ {+ "knowledge_bases": {+ "kb": {+ "path": "kb",+ "rag_provider": "lightrag",+ "status": "processing",+ }+ }+ }+ ),+ encoding="utf-8",+ )++ class _FailingRagService:+ def __init__(self, *_args, **_kwargs) -> None:+ pass++ async def initialize(self, *_args, **_kwargs) -> bool:+ raise RuntimeError("original indexing failure")++ rag_service_module = importlib.import_module("deeptutor.services.rag.service")+ monkeypatch.setattr(rag_service_module, "RAGService", _FailingRagService)++ class _ErrorWriteFailsTracker(knowledge_router_module.ProgressTracker):+ def update(self, stage, *args, **kwargs):+ if stage is knowledge_router_module.ProgressStage.ERROR:+ raise OSError("synthetic error progress write failure")+ return super().update(stage, *args, **kwargs)++ monkeypatch.setattr(knowledge_router_module, "ProgressTracker", _ErrorWriteFailsTracker)++ task_id = knowledge_router_module._build_unique_task_id("kb_reindex", "error-progress")+ with caplog.at_level(logging.WARNING, logger="deeptutor.api.routers.knowledge"):+ asyncio.run(+ knowledge_router_module.run_reindex_task(+ kb_name="kb",+ base_dir=str(base_dir),+ task_id=task_id,+ signature_hash="lightrag",+ )+ )++ task = knowledge_router_module.TaskIDManager.get_instance().get_task_metadata(task_id)+ assert task is not None+ assert task["status"] == "error"+ assert task["error"] == "original indexing failure"+ warnings = [+ record.getMessage() for record in caplog.records if record.levelno >= logging.WARNING+ ]+ assert any("error progress" in message and "kb" in message for message in warnings), warnings++@pytest.mark.parametrize("failed_sink", ["progress_file", "central_config"])def test_reindex_task_does_not_report_a_published_version_as_failed_when_bookkeeping_fails(monkeypatch, tmp_path: Path, failed_sink: strdiff --git a/tests/knowledge/test_linked_folder_sync.py b/tests/knowledge/test_linked_folder_sync.pyindex 14221b004..327c8dc63 100644--- a/tests/knowledge/test_linked_folder_sync.py+++ b/tests/knowledge/test_linked_folder_sync.py@@ -10,9 +10,11 @@ The atomicity contract of the shared writer lives infrom __future__ import annotationsimport json+import loggingfrom pathlib import Pathfrom deeptutor.api.routers.knowledge import LinkedFolderInfo+from deeptutor.knowledge import manager as manager_modulefrom deeptutor.knowledge.manager import KnowledgeBaseManager@@ -88,6 +90,54 @@ def test_empty_successful_sync_records_last_sync_without_changing_file_state(assert after["synced_files"] == before["synced_files"]+def test_update_folder_sync_state_mtime_failure_is_visible_not_fake_synced(+ tmp_path: Path,+ caplog,+ monkeypatch,+) -> None:+ """A per-file mtime recording failure must surface, not vanish.++ ``update_folder_sync_state()`` used to swallow stat/fromtimestamp errors+ with ``except Exception: pass``. The sync then reported success while the+ file's state was never recorded, hiding the degradation. The failure must+ be logged and the file must stay un-recorded so the next scan re-syncs it.+ """+ manager, metadata_file, folder_id, doc = _manager_with_linked_folder(tmp_path)++ flavour = type(Path()) # PosixPath on POSIX, WindowsPath on Windows++ class _VanishedMidStatPath(flavour):+ """exists() wins the race, stat() loses it (source vanished mid-sync)."""++ def exists(self, **kwargs):+ return True++ def stat(self, **kwargs):+ raise OSError("source vanished before stat")++ monkeypatch.setattr(manager_module, "Path", _VanishedMidStatPath)++ with caplog.at_level(logging.WARNING, logger="deeptutor.knowledge.manager"):+ manager.update_folder_sync_state("kb", folder_id, [str(doc)])++ # Visibility: the swallowed failure is now a logged warning naming the file.+ warnings_for_doc = [+ record+ for record in caplog.records+ if record.levelno >= logging.WARNING and str(doc) in record.getMessage()+ ]+ assert warnings_for_doc, f"expected a logged warning mentioning {doc}, got none"++ # Fallback semantics: the file is not marked synced, so the next scan+ # re-detects it instead of trusting a fabricated "already synced" state.+ on_disk = json.loads(metadata_file.read_text(encoding="utf-8"))+ folder = on_disk["linked_folders"][0]+ assert folder["last_sync"]+ assert str(doc) not in folder["synced_files"]+ assert folder["file_count"] == 0+ assert manager.detect_folder_changes("kb", folder_id)["new_files"] == [str(doc)]++def test_unlink_removes_only_the_source_registration(tmp_path: Path) -> None:manager, _metadata_file, folder_id, _doc = _manager_with_linked_folder(tmp_path)kb_dir = manager.base_dir / "kb"diff --git a/tests/knowledge/test_manager_update_kb_status_failures.py b/tests/knowledge/test_manager_update_kb_status_failures.pynew file mode 100644index 000000000..74a6fc6d5--- /dev/null+++ b/tests/knowledge/test_manager_update_kb_status_failures.py@@ -0,0 +1,72 @@+"""Failure-visibility tests for ``KnowledgeBaseManager.update_kb_status``.++A bare ``except Exception: pass`` around the ready-transition metadata+refresh meant that when refreshing embedding signatures or+``index_versions`` failed, the failure left no trace and the persisted KB+entry silently diverged from the on-disk index state that the UI renders.++These tests pin the contract that (a) a metadata refresh failure is logged+but stays best-effort, (b) a config-persist failure is logged at error level+and still reaches the caller as an exception, and (c) a successful update+persists the new status.+"""++from __future__ import annotations++import logging+from pathlib import Path++import pytest++from deeptutor.knowledge.manager import KnowledgeBaseManager+++def test_successful_update_persists_status(tmp_path: Path) -> None:+ manager = KnowledgeBaseManager(base_dir=str(tmp_path))++ manager.update_kb_status(name="kb1", status="processing")++ assert manager.get_kb_status("kb1")["status"] == "processing"+++def test_persist_failure_is_logged_and_raised(+ tmp_path: Path, monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture+) -> None:+ manager = KnowledgeBaseManager(base_dir=str(tmp_path))++ def _fail_save() -> None:+ raise OSError("disk full")++ monkeypatch.setattr(manager, "_save_config", _fail_save)++ with caplog.at_level(logging.ERROR, logger="deeptutor.knowledge.manager"):+ with pytest.raises(OSError, match="disk full"):+ manager.update_kb_status(name="kb1", status="error")++ assert any(+ "kb1" in record.message and record.levelno == logging.ERROR for record in caplog.records+ )+++def test_ready_metadata_failure_is_logged(+ tmp_path: Path, monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture+) -> None:+ from deeptutor.knowledge import manager as manager_module++ manager = KnowledgeBaseManager(base_dir=str(tmp_path))+ (tmp_path / "kb1").mkdir()+ (tmp_path / "kb1" / "version-1").mkdir()++ def _boom(kb_dir: Path, provider: str) -> list[dict]:+ raise RuntimeError("docstore exploded")++ monkeypatch.setattr(manager_module, "inspect_kb_versions", _boom)++ with caplog.at_level(logging.WARNING, logger="deeptutor.knowledge.manager"):+ manager.update_kb_status(name="kb1", status="ready")++ # The metadata refresh is best-effort: the status write itself still lands.+ assert manager.get_kb_status("kb1")["status"] == "ready"+ assert any(+ "kb1" in record.message and record.levelno == logging.WARNING for record in caplog.records+ )