Skip to content

DeepTutor: Lifelong Personalized Tutoring. https://deeptutor.info/.

fix(runtime): log swallowed launcher, init and update-worker failures (#1704)

evan188199-tech3a192f6a5321
diff --git a/deeptutor/runtime/launcher.py b/deeptutor/runtime/launcher.py
index 968d8eab7..122d4543b 100644
--- a/deeptutor/runtime/launcher.py
+++ b/deeptutor/runtime/launcher.py
@@ -141,25 +141,41 @@ def _log(message: str) -> None:
def _reset_runtime_singletons() -> None:
- """Make a just-selected DEEPTUTOR_HOME visible to path/config singletons."""
+ """Make a just-selected DEEPTUTOR_HOME visible to path/config singletons.
+
+ Same caveat as ``deeptutor_cli.init_cmd``: a reset that silently fails
+ leaves the previous home's cached paths in place, so failures are logged
+ instead of swallowed.
+ """
try:
from deeptutor.services.path_service import PathService
PathService.reset_instance()
except Exception:
- pass
+ logger.exception(
+ "PathService reset failed; the runtime may keep writing to the "
+ "previous DEEPTUTOR_HOME instead of the selected one"
+ )
try:
from deeptutor.services.config.runtime_settings import RuntimeSettingsService
RuntimeSettingsService._instances.clear()
except Exception:
- pass
+ logger.warning(
+ "RuntimeSettings cache reset failed; settings cached for the "
+ "previous DEEPTUTOR_HOME may be reused",
+ exc_info=True,
+ )
try:
from deeptutor.services.config.model_catalog import ModelCatalogService
ModelCatalogService._instances.clear()
except Exception:
- pass
+ logger.warning(
+ "ModelCatalog cache reset failed; model catalog cached for the "
+ "previous DEEPTUTOR_HOME may be reused",
+ exc_info=True,
+ )
def _get_pgid(pid: int | None) -> int | None:
@@ -1291,8 +1307,11 @@ def _handoff_pending_update(
except Exception as exc:
try:
store.mark_failed(job.id, f"Launcher handoff failed: {exc}")
- except Exception:
- pass
+ except Exception as mark_exc:
+ _log(
+ f"Launcher handoff failed for update {job.id}: {exc} "
+ f"(could not record failure: {mark_exc})"
+ )
return False
return True
diff --git a/deeptutor/runtime/update_worker.py b/deeptutor/runtime/update_worker.py
index fb45320b1..baf040486 100644
--- a/deeptutor/runtime/update_worker.py
+++ b/deeptutor/runtime/update_worker.py
@@ -100,6 +100,26 @@ def _launch_restart(job: UpdateJob, *, log_path: Path) -> None:
)
+def _log_aborted_recovery(
+ store: UpdateJobStore, *, original: Exception, load_error: Exception
+) -> None:
+ """Leave a durable record when recovery cannot even read the job state."""
+
+ message = (
+ "update worker: recovery aborted; update state could not be loaded "
+ f"({type(load_error).__name__}: {load_error}). "
+ f"Original failure: {type(original).__name__}: {original}. "
+ "The job state was left untouched; inspect it before restarting DeepTutor.\n"
+ )
+ try:
+ with store.log_path.open("a", encoding="utf-8") as log:
+ log.write(message)
+ except OSError:
+ # The launcher redirects worker stderr into this same log file; if the
+ # file itself is unreachable there is nowhere durable left to report.
+ print(message, file=sys.stderr)
+
+
def run_update_worker(
*,
store_root: Path,
@@ -134,6 +154,13 @@ def run_update_worker(
except Exception as exc:
try:
current = store.load()
+ except Exception as load_error:
+ # Corrupt or unreadable state: both mark_failed and the trusted
+ # restart vector come from it, so record why and skip this round
+ # instead of exiting silently.
+ _log_aborted_recovery(store, original=exc, load_error=load_error)
+ return 1
+ try:
if current.status not in {"succeeded", "failed"}:
current = store.mark_failed(current.id, str(exc) or type(exc).__name__)
if current.restart_home and current.restart_argv:
diff --git a/deeptutor_cli/init_cmd.py b/deeptutor_cli/init_cmd.py
index 7d0db587a..ec5474bb4 100644
--- a/deeptutor_cli/init_cmd.py
+++ b/deeptutor_cli/init_cmd.py
@@ -10,6 +10,7 @@ intentionally thin so the order of steps is easy to read top-to-bottom.
from __future__ import annotations
+import logging
from pathlib import Path
from typing import Any
@@ -20,32 +21,47 @@ from deeptutor.runtime.home import DEEPTUTOR_HOME_ENV, get_runtime_home
from . import init_wizard as wiz
+logger = logging.getLogger(__name__)
+
def _reset_runtime_singletons() -> None:
"""Drop cached service instances so the new DEEPTUTOR_HOME takes effect.
``deeptutor init`` may pass ``--home`` to target a different workspace; the
singletons cache paths from the *previous* PathService and will silently
- write to the wrong place if not cleared.
+ write to the wrong place if not cleared. Failures are therefore logged
+ instead of swallowed: a silent reset failure means every later write still
+ lands in the old workspace with no trace of why.
"""
try:
from deeptutor.services.path_service import PathService
PathService.reset_instance()
except Exception:
- pass
+ logger.exception(
+ "PathService reset failed; later writes may keep going to the "
+ "previous DEEPTUTOR_HOME instead of the selected one"
+ )
try:
from deeptutor.services.config.runtime_settings import RuntimeSettingsService
RuntimeSettingsService._instances.clear()
except Exception:
- pass
+ logger.warning(
+ "RuntimeSettings cache reset failed; settings cached for the "
+ "previous DEEPTUTOR_HOME may be reused",
+ exc_info=True,
+ )
try:
from deeptutor.services.config.model_catalog import ModelCatalogService
ModelCatalogService._instances.clear()
except Exception:
- pass
+ logger.warning(
+ "ModelCatalog cache reset failed; model catalog cached for the "
+ "previous DEEPTUTOR_HOME may be reused",
+ exc_info=True,
+ )
def _ensure_model_service(catalog: dict, service_name: str, profile_id: str, model_id: str):
diff --git a/tests/cli/test_init_cmd_reset_singletons.py b/tests/cli/test_init_cmd_reset_singletons.py
new file mode 100644
index 000000000..11f67a3d1
--- /dev/null
+++ b/tests/cli/test_init_cmd_reset_singletons.py
@@ -0,0 +1,122 @@
+"""Singleton reset failures during ``deeptutor init`` must be logged, not swallowed.
+
+Regression coverage for the swallowed-error scan HIGH finding at
+``deeptutor_cli/init_cmd.py``: a failed ``PathService`` reset leaves the
+previous home's cached paths in place, so ``deeptutor init --home`` would
+silently write into the *old* workspace. The reset helper must log every
+failure explicitly instead of passing silently.
+"""
+
+from __future__ import annotations
+
+import logging
+from typing import Any
+
+import pytest
+
+
+class _ExplodingClearDict(dict):
+ """Dict whose ``clear`` always fails, simulating a broken cache reset."""
+
+ def clear(self) -> None:
+ raise RuntimeError("cache clear exploded")
+
+
+class _RecordingClearDict(dict):
+ """Dict that records whether ``clear`` was actually attempted."""
+
+ def __init__(self, *args: Any, calls: list[str], label: str) -> None:
+ super().__init__(*args)
+ self.calls = calls
+ self.label = label
+
+ def clear(self) -> None:
+ self.calls.append(self.label)
+ super().clear()
+
+
+def _fail_reset(monkeypatch: pytest.MonkeyPatch, owner: type, name: str) -> None:
+ """Replace ``owner.name`` with a callable that always raises."""
+
+ def _boom(*_args: Any, **_kwargs: Any) -> None:
+ raise RuntimeError("reset exploded")
+
+ monkeypatch.setattr(owner, name, _boom)
+
+
+def test_pathservice_reset_failure_is_logged(monkeypatch: pytest.MonkeyPatch, caplog) -> None:
+ """A failed PathService reset must surface at ERROR level, not vanish."""
+ from deeptutor.services.path_service import PathService
+ from deeptutor_cli import init_cmd
+
+ _fail_reset(monkeypatch, PathService, "reset_instance")
+
+ with caplog.at_level(logging.ERROR):
+ init_cmd._reset_runtime_singletons()
+
+ records = [record for record in caplog.records if "PathService" in record.getMessage()]
+ assert records, "failed PathService reset was swallowed silently"
+ assert any(record.levelno >= logging.ERROR for record in records)
+
+
+def test_runtime_settings_cache_reset_failure_is_logged(
+ monkeypatch: pytest.MonkeyPatch, caplog
+) -> None:
+ """A failed RuntimeSettings cache clear must be logged, not swallowed."""
+ from deeptutor.services.config.runtime_settings import RuntimeSettingsService
+ from deeptutor_cli import init_cmd
+
+ monkeypatch.setattr(
+ RuntimeSettingsService, "_instances", _ExplodingClearDict({"stale": object()})
+ )
+
+ with caplog.at_level(logging.WARNING):
+ init_cmd._reset_runtime_singletons()
+
+ records = [record for record in caplog.records if "RuntimeSettings" in record.getMessage()]
+ assert records, "failed RuntimeSettings cache clear was swallowed silently"
+ assert any(record.levelno >= logging.WARNING for record in records)
+
+
+def test_model_catalog_cache_reset_failure_is_logged(
+ monkeypatch: pytest.MonkeyPatch, caplog
+) -> None:
+ """A failed ModelCatalog cache clear must be logged, not swallowed."""
+ from deeptutor.services.config.model_catalog import ModelCatalogService
+ from deeptutor_cli import init_cmd
+
+ monkeypatch.setattr(ModelCatalogService, "_instances", _ExplodingClearDict({"stale": object()}))
+
+ with caplog.at_level(logging.WARNING):
+ init_cmd._reset_runtime_singletons()
+
+ records = [record for record in caplog.records if "ModelCatalog" in record.getMessage()]
+ assert records, "failed ModelCatalog cache clear was swallowed silently"
+ assert any(record.levelno >= logging.WARNING for record in records)
+
+
+def test_pathservice_failure_does_not_block_cache_resets(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ """Each reset is best-effort: one failure must not skip the other caches."""
+ from deeptutor.services.config.model_catalog import ModelCatalogService
+ from deeptutor.services.config.runtime_settings import RuntimeSettingsService
+ from deeptutor.services.path_service import PathService
+ from deeptutor_cli import init_cmd
+
+ calls: list[str] = []
+ _fail_reset(monkeypatch, PathService, "reset_instance")
+ monkeypatch.setattr(
+ RuntimeSettingsService,
+ "_instances",
+ _RecordingClearDict(calls=calls, label="runtime_settings"),
+ )
+ monkeypatch.setattr(
+ ModelCatalogService,
+ "_instances",
+ _RecordingClearDict(calls=calls, label="model_catalog"),
+ )
+
+ init_cmd._reset_runtime_singletons()
+
+ assert calls == ["runtime_settings", "model_catalog"]
diff --git a/tests/runtime/test_launcher.py b/tests/runtime/test_launcher.py
index f95eec796..bee8a9997 100644
--- a/tests/runtime/test_launcher.py
+++ b/tests/runtime/test_launcher.py
@@ -14,7 +14,7 @@ import pytest
from deeptutor.runtime import launcher
from deeptutor.runtime import process as runtime_process
from deeptutor.runtime.home import validate_runtime_home
-from deeptutor.services.app_update import UpdateJobStore, update_store_root
+from deeptutor.services.app_update import UpdateJob, UpdateJobStore, update_store_root
class _FakeTty:
@@ -191,6 +191,42 @@ def test_launcher_keeps_systemd_service_running_for_pending_update(
assert not store.active_path.exists()
+def test_launcher_handoff_failure_still_logged_when_marking_fails(
+ monkeypatch: pytest.MonkeyPatch, tmp_path: Path
+) -> None:
+ """A handoff failure must leave a trace even if mark_failed fails too."""
+
+ from deeptutor.services import app_update
+
+ monkeypatch.setattr(app_update, "running_under_systemd_service", lambda: False)
+ store = UpdateJobStore(update_store_root(tmp_path))
+ store.create(current_version="1.6.1", target_version="1.7.0")
+
+ def failing_worker_launcher(_root: Path) -> None:
+ raise RuntimeError("worker spawn boom")
+
+ def broken_mark_failed(_self: UpdateJobStore, _job_id: str, _error: str) -> UpdateJob:
+ raise OSError("state write boom")
+
+ monkeypatch.setattr(UpdateJobStore, "mark_failed", broken_mark_failed)
+ logs: list[str] = []
+ monkeypatch.setattr(launcher, "_log", logs.append)
+
+ assert (
+ launcher._handoff_pending_update(
+ tmp_path,
+ restart_argv=["start", "--home", str(tmp_path.resolve())],
+ worker_launcher=failing_worker_launcher,
+ )
+ is False
+ )
+ assert logs, "double handoff failure vanished without any log"
+ trace = " | ".join(logs)
+ assert "Launcher handoff failed" in trace
+ assert "worker spawn boom" in trace
+ assert "state write boom" in trace
+
+
def test_launcher_completes_update_only_after_restart(tmp_path: Path) -> None:
store = UpdateJobStore(update_store_root(tmp_path))
pending = store.create(current_version="1.6.1", target_version="1.7.0")
diff --git a/tests/runtime/test_launcher_reset_singletons.py b/tests/runtime/test_launcher_reset_singletons.py
new file mode 100644
index 000000000..68344626d
--- /dev/null
+++ b/tests/runtime/test_launcher_reset_singletons.py
@@ -0,0 +1,123 @@
+"""Singleton reset failures in the launcher must be logged, not swallowed.
+
+Regression coverage for the swallowed-error scan HIGH finding at
+``deeptutor/runtime/launcher.py`` (second copy of the same logic as
+``deeptutor_cli/init_cmd.py``): a failed ``PathService`` reset leaves the
+previous home's cached paths in place, so a runtime started against a new
+``DEEPTUTOR_HOME`` would silently write into the *old* workspace. The reset
+helper must log every failure explicitly instead of passing silently.
+"""
+
+from __future__ import annotations
+
+import logging
+from typing import Any
+
+import pytest
+
+
+class _ExplodingClearDict(dict):
+ """Dict whose ``clear`` always fails, simulating a broken cache reset."""
+
+ def clear(self) -> None:
+ raise RuntimeError("cache clear exploded")
+
+
+class _RecordingClearDict(dict):
+ """Dict that records whether ``clear`` was actually attempted."""
+
+ def __init__(self, *args: Any, calls: list[str], label: str) -> None:
+ super().__init__(*args)
+ self.calls = calls
+ self.label = label
+
+ def clear(self) -> None:
+ self.calls.append(self.label)
+ super().clear()
+
+
+def _fail_reset(monkeypatch: pytest.MonkeyPatch, owner: type, name: str) -> None:
+ """Replace ``owner.name`` with a callable that always raises."""
+
+ def _boom(*_args: Any, **_kwargs: Any) -> None:
+ raise RuntimeError("reset exploded")
+
+ monkeypatch.setattr(owner, name, _boom)
+
+
+def test_pathservice_reset_failure_is_logged(monkeypatch: pytest.MonkeyPatch, caplog) -> None:
+ """A failed PathService reset must surface at ERROR level, not vanish."""
+ from deeptutor.runtime import launcher
+ from deeptutor.services.path_service import PathService
+
+ _fail_reset(monkeypatch, PathService, "reset_instance")
+
+ with caplog.at_level(logging.ERROR):
+ launcher._reset_runtime_singletons()
+
+ records = [record for record in caplog.records if "PathService" in record.getMessage()]
+ assert records, "failed PathService reset was swallowed silently"
+ assert any(record.levelno >= logging.ERROR for record in records)
+
+
+def test_runtime_settings_cache_reset_failure_is_logged(
+ monkeypatch: pytest.MonkeyPatch, caplog
+) -> None:
+ """A failed RuntimeSettings cache clear must be logged, not swallowed."""
+ from deeptutor.runtime import launcher
+ from deeptutor.services.config.runtime_settings import RuntimeSettingsService
+
+ monkeypatch.setattr(
+ RuntimeSettingsService, "_instances", _ExplodingClearDict({"stale": object()})
+ )
+
+ with caplog.at_level(logging.WARNING):
+ launcher._reset_runtime_singletons()
+
+ records = [record for record in caplog.records if "RuntimeSettings" in record.getMessage()]
+ assert records, "failed RuntimeSettings cache clear was swallowed silently"
+ assert any(record.levelno >= logging.WARNING for record in records)
+
+
+def test_model_catalog_cache_reset_failure_is_logged(
+ monkeypatch: pytest.MonkeyPatch, caplog
+) -> None:
+ """A failed ModelCatalog cache clear must be logged, not swallowed."""
+ from deeptutor.runtime import launcher
+ from deeptutor.services.config.model_catalog import ModelCatalogService
+
+ monkeypatch.setattr(ModelCatalogService, "_instances", _ExplodingClearDict({"stale": object()}))
+
+ with caplog.at_level(logging.WARNING):
+ launcher._reset_runtime_singletons()
+
+ records = [record for record in caplog.records if "ModelCatalog" in record.getMessage()]
+ assert records, "failed ModelCatalog cache clear was swallowed silently"
+ assert any(record.levelno >= logging.WARNING for record in records)
+
+
+def test_pathservice_failure_does_not_block_cache_resets(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ """Each reset is best-effort: one failure must not skip the other caches."""
+ from deeptutor.runtime import launcher
+ from deeptutor.services.config.model_catalog import ModelCatalogService
+ from deeptutor.services.config.runtime_settings import RuntimeSettingsService
+ from deeptutor.services.path_service import PathService
+
+ calls: list[str] = []
+ _fail_reset(monkeypatch, PathService, "reset_instance")
+ monkeypatch.setattr(
+ RuntimeSettingsService,
+ "_instances",
+ _RecordingClearDict(calls=calls, label="runtime_settings"),
+ )
+ monkeypatch.setattr(
+ ModelCatalogService,
+ "_instances",
+ _RecordingClearDict(calls=calls, label="model_catalog"),
+ )
+
+ launcher._reset_runtime_singletons()
+
+ assert calls == ["runtime_settings", "model_catalog"]
diff --git a/tests/runtime/test_update_worker.py b/tests/runtime/test_update_worker.py
index 4463ce10e..7aadab2cd 100644
--- a/tests/runtime/test_update_worker.py
+++ b/tests/runtime/test_update_worker.py
@@ -66,6 +66,32 @@ def test_worker_failure_is_durable_and_restores_app(tmp_path: Path) -> None:
assert restarted and restarted[0].status == "failed"
+def test_worker_recovery_reports_and_skips_on_corrupt_state(tmp_path: Path) -> None:
+ store, _job = _handoff_job(tmp_path)
+ restarted: list[UpdateJob] = []
+ corrupt_state = "{ not valid json"
+
+ def run_and_corrupt(command: list[str], cwd: Path, log_path: Path) -> int:
+ store.state_path.write_text(corrupt_state, encoding="utf-8")
+ return 7
+
+ result = run_update_worker(
+ store_root=store.root,
+ parent_pid=123,
+ wait_for_parent=lambda _pid: None,
+ command_runner=run_and_corrupt,
+ restart_launcher=lambda value, _log: restarted.append(value),
+ )
+
+ assert result == 1
+ assert restarted == []
+ assert store.state_path.read_text(encoding="utf-8") == corrupt_state
+ log = store.log_path.read_text(encoding="utf-8")
+ assert "recovery aborted" in log
+ assert "could not be loaded" in log
+ assert "pip exited with status 7" in log
+
+
def test_update_command_rejects_non_stable_or_injected_versions() -> None:
with pytest.raises(ValueError):
build_update_command("1.7.0rc1")