Skip to content

DeepTutor: Lifelong Personalized Tutoring. https://deeptutor.info/.

fix: log silently swallowed failures in five services (#1707)

evan188199-tech0802510450d8
diff --git a/deeptutor/api/routers/question.py b/deeptutor/api/routers/question.py
index d996beb30..62defd9fc 100644
--- a/deeptutor/api/routers/question.py
+++ b/deeptutor/api/routers/question.py
@@ -320,8 +320,10 @@ async def websocket_mimic_generate(websocket: WebSocket):
error_msg = format_exception_message(e)
try:
await websocket.send_json({"type": "error", "content": error_msg})
+ except (RuntimeError, WebSocketDisconnect):
+ logger.debug("WebSocket closed, cannot send error event")
except Exception:
- pass
+ logger.warning("Failed to send error event to client", exc_info=True)
finally:
# Ensure stdout is always restored
sys.stdout = original_stdout
diff --git a/deeptutor/services/codex_auth/service.py b/deeptutor/services/codex_auth/service.py
index a3a9383de..d904b19fc 100644
--- a/deeptutor/services/codex_auth/service.py
+++ b/deeptutor/services/codex_auth/service.py
@@ -869,8 +869,16 @@ class CodexOAuthService:
if credentials is not None:
try:
await self._oauth.revoke(credentials)
- except Exception:
- pass
+ except Exception as exc:
+ # Never log credential material: record only the error
+ # kind and status so operators know the remote token
+ # may still be valid.
+ logger.warning(
+ "Codex token revocation failed during logout; the "
+ "remote token may still be valid (error=%s, http_status=%s)",
+ getattr(exc, "code", None) or type(exc).__name__,
+ getattr(exc, "http_status", None),
+ )
self._store.clear_credentials(expected_generation=credentials.generation)
else:
self._store.clear_credentials(
diff --git a/deeptutor/services/llm/provider_core/codebuddy_provider.py b/deeptutor/services/llm/provider_core/codebuddy_provider.py
index 670a8781b..0036b7d24 100644
--- a/deeptutor/services/llm/provider_core/codebuddy_provider.py
+++ b/deeptutor/services/llm/provider_core/codebuddy_provider.py
@@ -9,6 +9,7 @@ from contextlib import asynccontextmanager
from copy import deepcopy
from dataclasses import dataclass, field
import json
+import logging
import os
import re
import shutil
@@ -29,6 +30,16 @@ _SESSION_POOL_MAXSIZE = 4
_MCP_SERVER_NAME = "deeptutor"
_MCP_TOOL_PREFIX = f"mcp__{_MCP_SERVER_NAME}__"
+logger = logging.getLogger(__name__)
+
+
+def _short_exception(exc: BaseException) -> str:
+ """Type plus a short reason — never prompt bodies or credentials."""
+ reason = str(exc).strip()
+ if len(reason) > 200:
+ reason = reason[:200] + "..."
+ return reason or "<no message>"
+
@dataclass
class _SessionTurn:
@@ -549,9 +560,15 @@ async def _consume_messages(
if interrupt is not None:
try:
await interrupt()
+ except Exception as exc:
+ logger.warning(
+ "CodeBuddy interrupt after tool calls failed (%s: %s); "
+ "background generation may keep running",
+ type(exc).__name__,
+ _short_exception(exc),
+ )
+ else:
await _drain_interrupted_response(messages)
- except Exception:
- pass
return LLMResponse(
content="".join(chunks),
tool_calls=tool_calls,
@@ -571,8 +588,13 @@ async def _drain_interrupted_response(messages: Any) -> None:
async for message in messages:
if _is_result_message(message) or type(message).__name__ == "ErrorMessage":
return
- except Exception:
- return
+ except Exception as exc:
+ logger.warning(
+ "CodeBuddy interrupted response drain failed (%s: %s); "
+ "stale SDK messages may surface in the next turn",
+ type(exc).__name__,
+ _short_exception(exc),
+ )
def _assistant_tool_calls(message: Any) -> list[ToolCallRequest]:
diff --git a/deeptutor/services/memory/snapshot/adapters.py b/deeptutor/services/memory/snapshot/adapters.py
index 298cb8a49..6518168ce 100644
--- a/deeptutor/services/memory/snapshot/adapters.py
+++ b/deeptutor/services/memory/snapshot/adapters.py
@@ -130,7 +130,8 @@ def read_notebook_entities() -> list[Entity]:
continue
try:
nb_data = json.loads(nb_file.read_text(encoding="utf-8"))
- except (OSError, json.JSONDecodeError):
+ except (OSError, json.JSONDecodeError) as exc:
+ logger.warning("notebook snapshot skipped corrupt file: %s (%s)", nb_file, exc)
continue
for r in nb_data.get("records") or []:
if not isinstance(r, dict):
@@ -181,7 +182,8 @@ def read_cowriter_entities() -> list[Entity]:
continue
try:
m = json.loads(manifest.read_text(encoding="utf-8"))
- except (OSError, json.JSONDecodeError):
+ except (OSError, json.JSONDecodeError) as exc:
+ logger.warning("cowriter snapshot skipped corrupt manifest: %s (%s)", manifest, exc)
continue
doc_id = m.get("id")
if not doc_id:
@@ -214,7 +216,8 @@ def read_book_entities() -> list[Entity]:
continue
try:
m = json.loads(manifest_path.read_text(encoding="utf-8"))
- except (OSError, json.JSONDecodeError):
+ except (OSError, json.JSONDecodeError) as exc:
+ logger.warning("book snapshot skipped corrupt manifest: %s (%s)", manifest_path, exc)
continue
book_id = m.get("id")
if not book_id:
diff --git a/deeptutor/services/notebook/service.py b/deeptutor/services/notebook/service.py
index 4f1687781..1177ab303 100644
--- a/deeptutor/services/notebook/service.py
+++ b/deeptutor/services/notebook/service.py
@@ -150,7 +150,15 @@ class NotebookManager:
try:
with open(path, encoding="utf-8") as f:
notebook = json.load(f)
- except Exception:
+ except Exception as exc:
+ # The file is left untouched on disk, so the damage stays
+ # visible and repairable: list_notebooks() re-adopts it as an
+ # ``unreadable`` entry instead of the notebook quietly vanishing.
+ logger.warning(
+ "notebook file %s is unreadable (%s); excluded from the rebuilt index",
+ path,
+ exc,
+ )
continue
entries.append(self._index_row(notebook))
return entries
diff --git a/tests/api/test_question_router.py b/tests/api/test_question_router.py
index 5ef71d57a..aa08caf57 100644
--- a/tests/api/test_question_router.py
+++ b/tests/api/test_question_router.py
@@ -3,6 +3,7 @@ from __future__ import annotations
import asyncio
from contextlib import contextmanager
import importlib
+import logging
from pathlib import Path
import sys
import types
@@ -186,10 +187,12 @@ class _FakeMimicWebSocket:
incoming: list[dict] | None = None,
fail_send_when=None,
disconnect_immediately: bool = False,
+ error_send_exc: BaseException | None = None,
) -> None:
self._incoming = list(incoming or [])
self._fail_send_when = fail_send_when
self._disconnect_immediately = disconnect_immediately
+ self._error_send_exc = error_send_exc
self.accepted = False
self.sent: list[dict] = []
self.closed = False
@@ -205,6 +208,8 @@ class _FakeMimicWebSocket:
raise WebSocketDisconnect(code=1000)
async def send_json(self, payload: dict) -> None:
+ if payload.get("type") == "error" and self._error_send_exc is not None:
+ raise self._error_send_exc
if self._fail_send_when is not None and self._fail_send_when(payload):
raise RuntimeError("connection is closed")
self.sent.append(payload)
@@ -420,3 +425,87 @@ async def test_mimic_log_pusher_send_failure_stops_stream_without_breaking_run(
assert [message["type"] for message in websocket.sent] == ["status", "status", "complete"]
assert websocket.closed
assert sys.stdout is terminal
+
+
+async def _run_mimic_endpoint_until_error_send(
+ question_router_module, monkeypatch: pytest.MonkeyPatch, tmp_path: Path, error_send_exc
+) -> _FakeMimicWebSocket:
+ async def _raising_mimic_exam_questions(*_args, **_kwargs):
+ raise RuntimeError("mimic workflow exploded")
+
+ monkeypatch.setattr(
+ question_router_module, "mimic_exam_questions", _raising_mimic_exam_questions
+ )
+ monkeypatch.setattr(
+ question_router_module, "_mimic_output_dir", lambda: tmp_path / "mimic_papers"
+ )
+
+ async def _anonymous_ws_auth(_websocket):
+ return None
+
+ monkeypatch.setattr("deeptutor.api.routers.auth.ws_require_auth", _anonymous_ws_auth)
+
+ websocket = _FakeMimicWebSocket(
+ incoming=[_parsed_mode_config(tmp_path)],
+ error_send_exc=error_send_exc,
+ )
+ await question_router_module.websocket_mimic_generate(websocket)
+ return websocket
+
+
+async def test_mimic_error_event_send_failure_is_logged(
+ monkeypatch: pytest.MonkeyPatch,
+ tmp_path: Path,
+ caplog: pytest.LogCaptureFixture,
+) -> None:
+ question_router_module = _load_question_router_module(monkeypatch)
+
+ with caplog.at_level(logging.DEBUG, logger="deeptutor.api.routers.question"):
+ websocket = await _run_mimic_endpoint_until_error_send(
+ question_router_module, monkeypatch, tmp_path, ConnectionError("peer reset")
+ )
+
+ assert websocket.closed
+ # The original workflow failure is still logged server-side.
+ assert any(
+ record.levelno == logging.ERROR and "Mimic generation error" in record.getMessage()
+ for record in caplog.records
+ )
+ # The secondary failure to deliver the error event must not be swallowed.
+ assert any(
+ record.levelno >= logging.WARNING and "Failed to send error event" in record.getMessage()
+ for record in caplog.records
+ )
+
+
+ "send_exc",
+ [
+ WebSocketDisconnect(code=1001),
+ RuntimeError('Cannot call "send" once a close message has been sent.'),
+ ],
+)
+async def test_mimic_error_event_send_failure_on_closed_socket_is_not_an_error(
+ monkeypatch: pytest.MonkeyPatch,
+ tmp_path: Path,
+ caplog: pytest.LogCaptureFixture,
+ send_exc: BaseException,
+) -> None:
+ question_router_module = _load_question_router_module(monkeypatch)
+
+ with caplog.at_level(logging.DEBUG, logger="deeptutor.api.routers.question"):
+ await _run_mimic_endpoint_until_error_send(
+ question_router_module, monkeypatch, tmp_path, send_exc
+ )
+
+ # A closed/disconnected client is expected during teardown, not a warning.
+ assert not any(
+ record.levelno >= logging.WARNING and "Failed to send error event" in record.getMessage()
+ for record in caplog.records
+ )
+ assert any(
+ record.levelno == logging.DEBUG and "WebSocket closed" in record.getMessage()
+ for record in caplog.records
+ )
diff --git a/tests/services/codex_auth/test_service.py b/tests/services/codex_auth/test_service.py
index e109598bf..6f52184e7 100644
--- a/tests/services/codex_auth/test_service.py
+++ b/tests/services/codex_auth/test_service.py
@@ -4,6 +4,7 @@ import asyncio
from copy import deepcopy
import hashlib
import json
+import logging
from pathlib import Path
from typing import Any
from urllib.parse import parse_qs, urlsplit
@@ -1623,6 +1624,37 @@ async def test_revoke_failure_does_not_block_local_logout_and_restore(
assert _selection(model_catalog.load()) == _selection(original)
+async def test_revoke_failure_logs_warning_without_token_content(
+ tmp_path: Path,
+ caplog: pytest.LogCaptureFixture,
+) -> None:
+ service, _callback, oauth, _catalog, store, _model_catalog = await _oauth_service(tmp_path)
+ store.commit_credentials(_stored_credentials(), expected_generation=0)
+ oauth.revoke_error = CodexAuthError(
+ "token_revoke_failed",
+ "Codex authentication could not be revoked remotely.",
+ 502,
+ )
+
+ with caplog.at_level(logging.WARNING, logger="deeptutor.services.codex_auth.service"):
+ status = await service.logout()
+
+ assert status["connection"] == "disconnected"
+ assert store.load_credentials() is None
+ warnings = [
+ record
+ for record in caplog.records
+ if record.name == "deeptutor.services.codex_auth.service"
+ and record.levelno >= logging.WARNING
+ ]
+ assert warnings, "a failed token revocation during logout must be logged"
+ rendered = "\n".join(record.getMessage() for record in warnings)
+ assert "token_revoke_failed" in rendered
+ for secret in ("old-access", "old-refresh", "old-id", "account-123"):
+ assert secret not in rendered
+
+
@pytest.mark.asyncio
async def test_restarted_service_restores_connection_without_operation_or_secrets(
tmp_path: Path,
diff --git a/tests/services/llm/test_codebuddy_provider.py b/tests/services/llm/test_codebuddy_provider.py
index aafb02fda..6ac78a210 100644
--- a/tests/services/llm/test_codebuddy_provider.py
+++ b/tests/services/llm/test_codebuddy_provider.py
@@ -1,6 +1,7 @@
from __future__ import annotations
import asyncio
+import logging
import os
import shutil
import sys
@@ -399,6 +400,95 @@ async def test_codebuddy_session_drains_interrupt_before_tool_result_round(monke
await provider.aclose()
+async def test_codebuddy_interrupt_failure_after_tool_calls_logs_warning(
+ monkeypatch, caplog
+) -> None:
+ """A failed interrupt after tool_calls must warn, not vanish silently.
+
+ The SDK keeps generating in the background when interrupt() fails; without
+ a log the operator only sees the quota drain.
+ """
+
+ class FakeClient:
+ def __init__(self, options):
+ self.options = options
+ self.response_index = 0
+ self.interrupt_calls = 0
+
+ async def connect(self):
+ pass
+
+ async def query(self, _prompt):
+ self.response_index += 1
+
+ async def receive_response(self):
+ yield FakeAssistantMessage(
+ [
+ FakeToolUseBlock(
+ "tool-1",
+ "mcp__deeptutor__web_search",
+ {"query": "latest news"},
+ )
+ ]
+ )
+
+ async def interrupt(self):
+ self.interrupt_calls += 1
+ raise RuntimeError("interrupt pipe closed")
+
+ async def disconnect(self):
+ pass
+
+ monkeypatch.setitem(
+ sys.modules,
+ "codebuddy_agent_sdk",
+ SimpleNamespace(
+ query=lambda **_kwargs: None,
+ CodeBuddyAgentOptions=FakeOptions,
+ CodeBuddySDKClient=FakeClient,
+ tool=fake_tool,
+ create_sdk_mcp_server=fake_mcp_server,
+ ),
+ )
+ provider = CodeBuddyProvider()
+ tools = [
+ {
+ "type": "function",
+ "function": {
+ "name": "web_search",
+ "description": "Search the web",
+ "parameters": {"type": "object", "properties": {}},
+ },
+ }
+ ]
+
+ with caplog.at_level(
+ logging.WARNING,
+ logger="deeptutor.services.llm.provider_core.codebuddy_provider",
+ ):
+ response = await provider.chat(
+ [{"role": "user", "content": "Search"}],
+ tools=tools,
+ deeptutor_session_id="chat-interrupt-fail",
+ )
+
+ assert response.finish_reason == "tool_calls"
+ assert len(response.tool_calls) == 1
+ warnings = [
+ r
+ for r in caplog.records
+ if r.levelno >= logging.WARNING
+ and r.name == "deeptutor.services.llm.provider_core.codebuddy_provider"
+ ]
+ assert any(
+ "interrupt" in r.getMessage().lower() and "RuntimeError" in r.getMessage() for r in warnings
+ ), [r.getMessage() for r in warnings]
+ session = provider._sessions["chat-interrupt-fail"]
+ assert session.client.interrupt_calls == 1
+ await provider.aclose()
+
+
@pytest.mark.asyncio
async def test_fetch_codebuddy_models_uses_account_catalog(monkeypatch) -> None:
captured: dict[str, object] = {}
diff --git a/tests/services/memory/test_snapshot_adapters.py b/tests/services/memory/test_snapshot_adapters.py
index d1d3cfdd6..23f6e1097 100644
--- a/tests/services/memory/test_snapshot_adapters.py
+++ b/tests/services/memory/test_snapshot_adapters.py
@@ -10,6 +10,7 @@ any other surface, tagged with the originating partner.
from __future__ import annotations
import json
+import logging
from pathlib import Path
import pytest
@@ -169,3 +170,111 @@ def test_fingerprint_changes_when_conversation_grows(partner_tree: Path) -> None
)
fp2 = adapters.read_partner_entities()[0].fingerprint
assert fp1 != fp2
+
+
+# ── Corrupt-file tolerance: skip, warn (with path), keep the rest ────
+
+
+class _SnapshotPathService(_FakePathService):
+ """Fake exposing the notebook / co-writer / book path methods."""
+
+ def get_notebook_dir(self) -> Path:
+ return self.workspace_root / "notebook"
+
+ def get_notebook_file(self, notebook_id: str) -> Path:
+ return self.get_notebook_dir() / f"{notebook_id}.json"
+
+ def get_notebook_index_file(self) -> Path:
+ return self.get_notebook_dir() / "notebooks_index.json"
+
+ def get_co_writer_docs_dir(self) -> Path:
+ return self.workspace_root / "co_writer" / "documents"
+
+ def get_book_dir(self) -> Path:
+ return self.workspace_root / "book"
+
+
+def snapshot_tree(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path:
+ monkeypatch.setattr(adapters, "get_path_service", lambda: _SnapshotPathService(tmp_path))
+ return tmp_path
+
+
+def _warn_records(caplog: pytest.LogCaptureFixture) -> list[str]:
+ return [
+ rec.getMessage()
+ for rec in caplog.records
+ if rec.levelno == logging.WARNING
+ and rec.name == "deeptutor.services.memory.snapshot.adapters"
+ ]
+
+
+def test_corrupt_notebook_file_skipped_with_warning(
+ snapshot_tree: Path, caplog: pytest.LogCaptureFixture
+) -> None:
+ index = snapshot_tree / "notebook" / "notebooks_index.json"
+ index.parent.mkdir(parents=True)
+ index.write_text(
+ json.dumps({"notebooks": [{"id": "bad", "name": "Bad"}, {"id": "ok", "name": "Ok"}]}),
+ encoding="utf-8",
+ )
+ corrupt = snapshot_tree / "notebook" / "bad.json"
+ corrupt.write_text("{ not json", encoding="utf-8")
+ good = snapshot_tree / "notebook" / "ok.json"
+ good.write_text(
+ json.dumps({"records": [{"id": "r1", "title": "T", "output": "O"}]}),
+ encoding="utf-8",
+ )
+
+ with caplog.at_level(logging.WARNING):
+ entities = adapters.read_notebook_entities()
+
+ assert [e.id for e in entities] == ["r1"]
+ warnings = _warn_records(caplog)
+ assert any(str(corrupt) in w for w in warnings)
+
+
+def test_corrupt_cowriter_manifest_skipped_with_warning(
+ snapshot_tree: Path, caplog: pytest.LogCaptureFixture
+) -> None:
+ docs = snapshot_tree / "co_writer" / "documents"
+ bad_dir = docs / "doc_bad"
+ bad_dir.mkdir(parents=True)
+ corrupt = bad_dir / "manifest.json"
+ corrupt.write_text("{ oops", encoding="utf-8")
+ good_dir = docs / "doc_ok"
+ good_dir.mkdir()
+ (good_dir / "manifest.json").write_text(
+ json.dumps({"id": "ok", "title": "Doc", "content": "C"}),
+ encoding="utf-8",
+ )
+
+ with caplog.at_level(logging.WARNING):
+ entities = adapters.read_cowriter_entities()
+
+ assert [e.id for e in entities] == ["ok"]
+ warnings = _warn_records(caplog)
+ assert any(str(corrupt) in w for w in warnings)
+
+
+def test_corrupt_book_manifest_skipped_with_warning(
+ snapshot_tree: Path, caplog: pytest.LogCaptureFixture
+) -> None:
+ books = snapshot_tree / "book"
+ bad_dir = books / "book_bad"
+ bad_dir.mkdir(parents=True)
+ corrupt = bad_dir / "manifest.json"
+ corrupt.write_text("[ broken", encoding="utf-8")
+ good_dir = books / "book_ok"
+ good_dir.mkdir()
+ (good_dir / "manifest.json").write_text(
+ json.dumps({"id": "ok", "title": "Book", "description": "D"}),
+ encoding="utf-8",
+ )
+
+ with caplog.at_level(logging.WARNING):
+ entities = adapters.read_book_entities()
+
+ assert [e.id for e in entities] == ["ok"]
+ warnings = _warn_records(caplog)
+ assert any(str(corrupt) in w for w in warnings)
diff --git a/tests/services/test_notebook_service.py b/tests/services/test_notebook_service.py
index abcb0c6f9..bc0cc4428 100644
--- a/tests/services/test_notebook_service.py
+++ b/tests/services/test_notebook_service.py
@@ -142,6 +142,32 @@ def test_damaged_notebook_is_reported_not_hidden(tmp_path) -> None:
manager.get_notebook("broken01")
+def test_index_rebuild_reports_unreadable_files_instead_of_silence(tmp_path, caplog) -> None:
+ """Rebuilding the index must name each damaged file, never skip it silently.
+
+ A notebook file that exists but does not parse used to vanish from the
+ rebuilt index with no per-file log, so the damage was invisible until a
+ user noticed the entry missing.
+ """
+ import logging
+
+ manager = NotebookManager(base_dir=str(tmp_path))
+ good_id = manager.create_notebook("Good")["id"]
+ (tmp_path / "broken01.json").write_text("{ not json", encoding="utf-8")
+ manager.index_file.write_text("not json at all", encoding="utf-8")
+
+ with caplog.at_level(logging.WARNING, logger="deeptutor.services.notebook.service"):
+ entries = manager._rebuild_index_entries()
+
+ assert good_id in {row["id"] for row in entries}
+ assert any("broken01.json" in record.getMessage() for record in caplog.records)
+
+ # The damaged file stays visible through the normal listing path too.
+ listed = {row["id"]: row for row in manager.list_notebooks()}
+ assert listed["broken01"]["unreadable"] is True
+ assert good_id in listed
+
+
def test_list_notebooks_adopts_files_missing_from_the_index(tmp_path) -> None:
"""A notebook whose index row was lost is still listed."""
manager = NotebookManager(base_dir=str(tmp_path))