DeepTutor: Lifelong Personalized Tutoring. https://deeptutor.info/.
Python ai-agentsai-tutorclawbotcli-tooldeepresearchinteractive-learninglearningmulti-agent-systemsrag
fix(runtime): log swallowed launcher, init and update-worker failures (#1704)
evan188199-tech3a192f6a5321
diff --git a/deeptutor/runtime/launcher.py b/deeptutor/runtime/launcher.pyindex 968d8eab7..122d4543b 100644--- a/deeptutor/runtime/launcher.py+++ b/deeptutor/runtime/launcher.py@@ -141,25 +141,41 @@ def _log(message: str) -> None:def _reset_runtime_singletons() -> None:- """Make a just-selected DEEPTUTOR_HOME visible to path/config singletons."""+ """Make a just-selected DEEPTUTOR_HOME visible to path/config singletons.++ Same caveat as ``deeptutor_cli.init_cmd``: a reset that silently fails+ leaves the previous home's cached paths in place, so failures are logged+ instead of swallowed.+ """try:from deeptutor.services.path_service import PathServicePathService.reset_instance()except Exception:- pass+ logger.exception(+ "PathService reset failed; the runtime may keep writing to the "+ "previous DEEPTUTOR_HOME instead of the selected one"+ )try:from deeptutor.services.config.runtime_settings import RuntimeSettingsServiceRuntimeSettingsService._instances.clear()except Exception:- pass+ logger.warning(+ "RuntimeSettings cache reset failed; settings cached for the "+ "previous DEEPTUTOR_HOME may be reused",+ exc_info=True,+ )try:from deeptutor.services.config.model_catalog import ModelCatalogServiceModelCatalogService._instances.clear()except Exception:- pass+ logger.warning(+ "ModelCatalog cache reset failed; model catalog cached for the "+ "previous DEEPTUTOR_HOME may be reused",+ exc_info=True,+ )def _get_pgid(pid: int | None) -> int | None:@@ -1291,8 +1307,11 @@ def _handoff_pending_update(except Exception as exc:try:store.mark_failed(job.id, f"Launcher handoff failed: {exc}")- except Exception:- pass+ except Exception as mark_exc:+ _log(+ f"Launcher handoff failed for update {job.id}: {exc} "+ f"(could not record failure: {mark_exc})"+ )return Falsereturn Truediff --git a/deeptutor/runtime/update_worker.py b/deeptutor/runtime/update_worker.pyindex fb45320b1..baf040486 100644--- a/deeptutor/runtime/update_worker.py+++ b/deeptutor/runtime/update_worker.py@@ -100,6 +100,26 @@ def _launch_restart(job: UpdateJob, *, log_path: Path) -> None:)+def _log_aborted_recovery(+ store: UpdateJobStore, *, original: Exception, load_error: Exception+) -> None:+ """Leave a durable record when recovery cannot even read the job state."""++ message = (+ "update worker: recovery aborted; update state could not be loaded "+ f"({type(load_error).__name__}: {load_error}). "+ f"Original failure: {type(original).__name__}: {original}. "+ "The job state was left untouched; inspect it before restarting DeepTutor.\n"+ )+ try:+ with store.log_path.open("a", encoding="utf-8") as log:+ log.write(message)+ except OSError:+ # The launcher redirects worker stderr into this same log file; if the+ # file itself is unreachable there is nowhere durable left to report.+ print(message, file=sys.stderr)++def run_update_worker(*,store_root: Path,@@ -134,6 +154,13 @@ def run_update_worker(except Exception as exc:try:current = store.load()+ except Exception as load_error:+ # Corrupt or unreadable state: both mark_failed and the trusted+ # restart vector come from it, so record why and skip this round+ # instead of exiting silently.+ _log_aborted_recovery(store, original=exc, load_error=load_error)+ return 1+ try:if current.status not in {"succeeded", "failed"}:current = store.mark_failed(current.id, str(exc) or type(exc).__name__)if current.restart_home and current.restart_argv:diff --git a/deeptutor_cli/init_cmd.py b/deeptutor_cli/init_cmd.pyindex 7d0db587a..ec5474bb4 100644--- a/deeptutor_cli/init_cmd.py+++ b/deeptutor_cli/init_cmd.py@@ -10,6 +10,7 @@ intentionally thin so the order of steps is easy to read top-to-bottom.from __future__ import annotations+import loggingfrom pathlib import Pathfrom typing import Any@@ -20,32 +21,47 @@ from deeptutor.runtime.home import DEEPTUTOR_HOME_ENV, get_runtime_homefrom . import init_wizard as wiz+logger = logging.getLogger(__name__)+def _reset_runtime_singletons() -> None:"""Drop cached service instances so the new DEEPTUTOR_HOME takes effect.``deeptutor init`` may pass ``--home`` to target a different workspace; thesingletons cache paths from the *previous* PathService and will silently- write to the wrong place if not cleared.+ write to the wrong place if not cleared. Failures are therefore logged+ instead of swallowed: a silent reset failure means every later write still+ lands in the old workspace with no trace of why."""try:from deeptutor.services.path_service import PathServicePathService.reset_instance()except Exception:- pass+ logger.exception(+ "PathService reset failed; later writes may keep going to the "+ "previous DEEPTUTOR_HOME instead of the selected one"+ )try:from deeptutor.services.config.runtime_settings import RuntimeSettingsServiceRuntimeSettingsService._instances.clear()except Exception:- pass+ logger.warning(+ "RuntimeSettings cache reset failed; settings cached for the "+ "previous DEEPTUTOR_HOME may be reused",+ exc_info=True,+ )try:from deeptutor.services.config.model_catalog import ModelCatalogServiceModelCatalogService._instances.clear()except Exception:- pass+ logger.warning(+ "ModelCatalog cache reset failed; model catalog cached for the "+ "previous DEEPTUTOR_HOME may be reused",+ exc_info=True,+ )def _ensure_model_service(catalog: dict, service_name: str, profile_id: str, model_id: str):diff --git a/tests/cli/test_init_cmd_reset_singletons.py b/tests/cli/test_init_cmd_reset_singletons.pynew file mode 100644index 000000000..11f67a3d1--- /dev/null+++ b/tests/cli/test_init_cmd_reset_singletons.py@@ -0,0 +1,122 @@+"""Singleton reset failures during ``deeptutor init`` must be logged, not swallowed.++Regression coverage for the swallowed-error scan HIGH finding at+``deeptutor_cli/init_cmd.py``: a failed ``PathService`` reset leaves the+previous home's cached paths in place, so ``deeptutor init --home`` would+silently write into the *old* workspace. The reset helper must log every+failure explicitly instead of passing silently.+"""++from __future__ import annotations++import logging+from typing import Any++import pytest+++class _ExplodingClearDict(dict):+ """Dict whose ``clear`` always fails, simulating a broken cache reset."""++ def clear(self) -> None:+ raise RuntimeError("cache clear exploded")+++class _RecordingClearDict(dict):+ """Dict that records whether ``clear`` was actually attempted."""++ def __init__(self, *args: Any, calls: list[str], label: str) -> None:+ super().__init__(*args)+ self.calls = calls+ self.label = label++ def clear(self) -> None:+ self.calls.append(self.label)+ super().clear()+++def _fail_reset(monkeypatch: pytest.MonkeyPatch, owner: type, name: str) -> None:+ """Replace ``owner.name`` with a callable that always raises."""++ def _boom(*_args: Any, **_kwargs: Any) -> None:+ raise RuntimeError("reset exploded")++ monkeypatch.setattr(owner, name, _boom)+++def test_pathservice_reset_failure_is_logged(monkeypatch: pytest.MonkeyPatch, caplog) -> None:+ """A failed PathService reset must surface at ERROR level, not vanish."""+ from deeptutor.services.path_service import PathService+ from deeptutor_cli import init_cmd++ _fail_reset(monkeypatch, PathService, "reset_instance")++ with caplog.at_level(logging.ERROR):+ init_cmd._reset_runtime_singletons()++ records = [record for record in caplog.records if "PathService" in record.getMessage()]+ assert records, "failed PathService reset was swallowed silently"+ assert any(record.levelno >= logging.ERROR for record in records)+++def test_runtime_settings_cache_reset_failure_is_logged(+ monkeypatch: pytest.MonkeyPatch, caplog+) -> None:+ """A failed RuntimeSettings cache clear must be logged, not swallowed."""+ from deeptutor.services.config.runtime_settings import RuntimeSettingsService+ from deeptutor_cli import init_cmd++ monkeypatch.setattr(+ RuntimeSettingsService, "_instances", _ExplodingClearDict({"stale": object()})+ )++ with caplog.at_level(logging.WARNING):+ init_cmd._reset_runtime_singletons()++ records = [record for record in caplog.records if "RuntimeSettings" in record.getMessage()]+ assert records, "failed RuntimeSettings cache clear was swallowed silently"+ assert any(record.levelno >= logging.WARNING for record in records)+++def test_model_catalog_cache_reset_failure_is_logged(+ monkeypatch: pytest.MonkeyPatch, caplog+) -> None:+ """A failed ModelCatalog cache clear must be logged, not swallowed."""+ from deeptutor.services.config.model_catalog import ModelCatalogService+ from deeptutor_cli import init_cmd++ monkeypatch.setattr(ModelCatalogService, "_instances", _ExplodingClearDict({"stale": object()}))++ with caplog.at_level(logging.WARNING):+ init_cmd._reset_runtime_singletons()++ records = [record for record in caplog.records if "ModelCatalog" in record.getMessage()]+ assert records, "failed ModelCatalog cache clear was swallowed silently"+ assert any(record.levelno >= logging.WARNING for record in records)+++def test_pathservice_failure_does_not_block_cache_resets(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ """Each reset is best-effort: one failure must not skip the other caches."""+ from deeptutor.services.config.model_catalog import ModelCatalogService+ from deeptutor.services.config.runtime_settings import RuntimeSettingsService+ from deeptutor.services.path_service import PathService+ from deeptutor_cli import init_cmd++ calls: list[str] = []+ _fail_reset(monkeypatch, PathService, "reset_instance")+ monkeypatch.setattr(+ RuntimeSettingsService,+ "_instances",+ _RecordingClearDict(calls=calls, label="runtime_settings"),+ )+ monkeypatch.setattr(+ ModelCatalogService,+ "_instances",+ _RecordingClearDict(calls=calls, label="model_catalog"),+ )++ init_cmd._reset_runtime_singletons()++ assert calls == ["runtime_settings", "model_catalog"]diff --git a/tests/runtime/test_launcher.py b/tests/runtime/test_launcher.pyindex f95eec796..bee8a9997 100644--- a/tests/runtime/test_launcher.py+++ b/tests/runtime/test_launcher.py@@ -14,7 +14,7 @@ import pytestfrom deeptutor.runtime import launcherfrom deeptutor.runtime import process as runtime_processfrom deeptutor.runtime.home import validate_runtime_home-from deeptutor.services.app_update import UpdateJobStore, update_store_root+from deeptutor.services.app_update import UpdateJob, UpdateJobStore, update_store_rootclass _FakeTty:@@ -191,6 +191,42 @@ def test_launcher_keeps_systemd_service_running_for_pending_update(assert not store.active_path.exists()+def test_launcher_handoff_failure_still_logged_when_marking_fails(+ monkeypatch: pytest.MonkeyPatch, tmp_path: Path+) -> None:+ """A handoff failure must leave a trace even if mark_failed fails too."""++ from deeptutor.services import app_update++ monkeypatch.setattr(app_update, "running_under_systemd_service", lambda: False)+ store = UpdateJobStore(update_store_root(tmp_path))+ store.create(current_version="1.6.1", target_version="1.7.0")++ def failing_worker_launcher(_root: Path) -> None:+ raise RuntimeError("worker spawn boom")++ def broken_mark_failed(_self: UpdateJobStore, _job_id: str, _error: str) -> UpdateJob:+ raise OSError("state write boom")++ monkeypatch.setattr(UpdateJobStore, "mark_failed", broken_mark_failed)+ logs: list[str] = []+ monkeypatch.setattr(launcher, "_log", logs.append)++ assert (+ launcher._handoff_pending_update(+ tmp_path,+ restart_argv=["start", "--home", str(tmp_path.resolve())],+ worker_launcher=failing_worker_launcher,+ )+ is False+ )+ assert logs, "double handoff failure vanished without any log"+ trace = " | ".join(logs)+ assert "Launcher handoff failed" in trace+ assert "worker spawn boom" in trace+ assert "state write boom" in trace++def test_launcher_completes_update_only_after_restart(tmp_path: Path) -> None:store = UpdateJobStore(update_store_root(tmp_path))pending = store.create(current_version="1.6.1", target_version="1.7.0")diff --git a/tests/runtime/test_launcher_reset_singletons.py b/tests/runtime/test_launcher_reset_singletons.pynew file mode 100644index 000000000..68344626d--- /dev/null+++ b/tests/runtime/test_launcher_reset_singletons.py@@ -0,0 +1,123 @@+"""Singleton reset failures in the launcher must be logged, not swallowed.++Regression coverage for the swallowed-error scan HIGH finding at+``deeptutor/runtime/launcher.py`` (second copy of the same logic as+``deeptutor_cli/init_cmd.py``): a failed ``PathService`` reset leaves the+previous home's cached paths in place, so a runtime started against a new+``DEEPTUTOR_HOME`` would silently write into the *old* workspace. The reset+helper must log every failure explicitly instead of passing silently.+"""++from __future__ import annotations++import logging+from typing import Any++import pytest+++class _ExplodingClearDict(dict):+ """Dict whose ``clear`` always fails, simulating a broken cache reset."""++ def clear(self) -> None:+ raise RuntimeError("cache clear exploded")+++class _RecordingClearDict(dict):+ """Dict that records whether ``clear`` was actually attempted."""++ def __init__(self, *args: Any, calls: list[str], label: str) -> None:+ super().__init__(*args)+ self.calls = calls+ self.label = label++ def clear(self) -> None:+ self.calls.append(self.label)+ super().clear()+++def _fail_reset(monkeypatch: pytest.MonkeyPatch, owner: type, name: str) -> None:+ """Replace ``owner.name`` with a callable that always raises."""++ def _boom(*_args: Any, **_kwargs: Any) -> None:+ raise RuntimeError("reset exploded")++ monkeypatch.setattr(owner, name, _boom)+++def test_pathservice_reset_failure_is_logged(monkeypatch: pytest.MonkeyPatch, caplog) -> None:+ """A failed PathService reset must surface at ERROR level, not vanish."""+ from deeptutor.runtime import launcher+ from deeptutor.services.path_service import PathService++ _fail_reset(monkeypatch, PathService, "reset_instance")++ with caplog.at_level(logging.ERROR):+ launcher._reset_runtime_singletons()++ records = [record for record in caplog.records if "PathService" in record.getMessage()]+ assert records, "failed PathService reset was swallowed silently"+ assert any(record.levelno >= logging.ERROR for record in records)+++def test_runtime_settings_cache_reset_failure_is_logged(+ monkeypatch: pytest.MonkeyPatch, caplog+) -> None:+ """A failed RuntimeSettings cache clear must be logged, not swallowed."""+ from deeptutor.runtime import launcher+ from deeptutor.services.config.runtime_settings import RuntimeSettingsService++ monkeypatch.setattr(+ RuntimeSettingsService, "_instances", _ExplodingClearDict({"stale": object()})+ )++ with caplog.at_level(logging.WARNING):+ launcher._reset_runtime_singletons()++ records = [record for record in caplog.records if "RuntimeSettings" in record.getMessage()]+ assert records, "failed RuntimeSettings cache clear was swallowed silently"+ assert any(record.levelno >= logging.WARNING for record in records)+++def test_model_catalog_cache_reset_failure_is_logged(+ monkeypatch: pytest.MonkeyPatch, caplog+) -> None:+ """A failed ModelCatalog cache clear must be logged, not swallowed."""+ from deeptutor.runtime import launcher+ from deeptutor.services.config.model_catalog import ModelCatalogService++ monkeypatch.setattr(ModelCatalogService, "_instances", _ExplodingClearDict({"stale": object()}))++ with caplog.at_level(logging.WARNING):+ launcher._reset_runtime_singletons()++ records = [record for record in caplog.records if "ModelCatalog" in record.getMessage()]+ assert records, "failed ModelCatalog cache clear was swallowed silently"+ assert any(record.levelno >= logging.WARNING for record in records)+++def test_pathservice_failure_does_not_block_cache_resets(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ """Each reset is best-effort: one failure must not skip the other caches."""+ from deeptutor.runtime import launcher+ from deeptutor.services.config.model_catalog import ModelCatalogService+ from deeptutor.services.config.runtime_settings import RuntimeSettingsService+ from deeptutor.services.path_service import PathService++ calls: list[str] = []+ _fail_reset(monkeypatch, PathService, "reset_instance")+ monkeypatch.setattr(+ RuntimeSettingsService,+ "_instances",+ _RecordingClearDict(calls=calls, label="runtime_settings"),+ )+ monkeypatch.setattr(+ ModelCatalogService,+ "_instances",+ _RecordingClearDict(calls=calls, label="model_catalog"),+ )++ launcher._reset_runtime_singletons()++ assert calls == ["runtime_settings", "model_catalog"]diff --git a/tests/runtime/test_update_worker.py b/tests/runtime/test_update_worker.pyindex 4463ce10e..7aadab2cd 100644--- a/tests/runtime/test_update_worker.py+++ b/tests/runtime/test_update_worker.py@@ -66,6 +66,32 @@ def test_worker_failure_is_durable_and_restores_app(tmp_path: Path) -> None:assert restarted and restarted[0].status == "failed"+def test_worker_recovery_reports_and_skips_on_corrupt_state(tmp_path: Path) -> None:+ store, _job = _handoff_job(tmp_path)+ restarted: list[UpdateJob] = []+ corrupt_state = "{ not valid json"++ def run_and_corrupt(command: list[str], cwd: Path, log_path: Path) -> int:+ store.state_path.write_text(corrupt_state, encoding="utf-8")+ return 7++ result = run_update_worker(+ store_root=store.root,+ parent_pid=123,+ wait_for_parent=lambda _pid: None,+ command_runner=run_and_corrupt,+ restart_launcher=lambda value, _log: restarted.append(value),+ )++ assert result == 1+ assert restarted == []+ assert store.state_path.read_text(encoding="utf-8") == corrupt_state+ log = store.log_path.read_text(encoding="utf-8")+ assert "recovery aborted" in log+ assert "could not be loaded" in log+ assert "pip exited with status 7" in log++def test_update_command_rejects_non_stable_or_injected_versions() -> None:with pytest.raises(ValueError):build_update_command("1.7.0rc1")