DeepTutor: Lifelong Personalized Tutoring. https://deeptutor.info/.
Python ai-agentsai-tutorclawbotcli-tooldeepresearchinteractive-learninglearningmulti-agent-systemsrag
fix(services): surface fsync failures in atomic writers as rate-limited warnings (#1751)
evan188199-tech409fa3c4e14c
diff --git a/deeptutor/services/file_io.py b/deeptutor/services/file_io.pyindex bdf64c230..9ccc02be5 100644--- a/deeptutor/services/file_io.py+++ b/deeptutor/services/file_io.py@@ -3,12 +3,47 @@from __future__ import annotationsimport json+import loggingimport osfrom pathlib import Pathimport tempfile+import threadingimport timefrom typing import Any+logger = logging.getLogger(__name__)++# fsync failures weaken the durability promise but do not corrupt the write:+# the replacement below still publishes complete content. Warn about them,+# rate-limited, so a failing disk fills the log once per interval instead of+# once per write.+_FSYNC_WARNING_INTERVAL_SECONDS = 60.0+_last_fsync_warning_at: float | None = None+_fsync_warning_lock = threading.Lock()+++def _sync_to_disk(handle: Any, path: Path) -> None:+ """fsync *handle*, surfacing a failure as a rate-limited warning."""+ global _last_fsync_warning_at+ try:+ os.fsync(handle.fileno())+ except OSError as exc:+ now = time.monotonic()+ with _fsync_warning_lock:+ should_warn = (+ _last_fsync_warning_at is None+ or now - _last_fsync_warning_at >= _FSYNC_WARNING_INTERVAL_SECONDS+ )+ if should_warn:+ _last_fsync_warning_at = now+ if should_warn:+ logger.warning(+ "fsync failed while writing %s; the write succeeded but is not"+ " guaranteed durable on disk (%r)",+ path,+ exc,+ )+def _atomic_replace(src: Path, dst: Path, *, max_retries: int = 5) -> None:"""Replace *dst* with *src*, retrying on transient ``PermissionError``.@@ -50,10 +85,7 @@ def atomic_write_json(path: Path, payload: dict[str, Any]) -> None:json.dump(payload, handle, indent=2, ensure_ascii=False)handle.write("\n")handle.flush()- try:- os.fsync(handle.fileno())- except OSError:- pass+ _sync_to_disk(handle, path)_atomic_replace(temporary_path, path)finally:if temporary_path is not None:@@ -75,10 +107,7 @@ def atomic_write_text(path: Path, text: str) -> None:temporary_path = Path(handle.name)handle.write(text)handle.flush()- try:- os.fsync(handle.fileno())- except OSError:- pass+ _sync_to_disk(handle, path)_atomic_replace(temporary_path, path)finally:if temporary_path is not None:diff --git a/tests/services/test_file_io.py b/tests/services/test_file_io.pyindex 2c4107830..81e07a174 100644--- a/tests/services/test_file_io.py+++ b/tests/services/test_file_io.py@@ -1,7 +1,9 @@from __future__ import annotationsimport json+import loggingfrom pathlib import Path+import timeimport pytest@@ -9,6 +11,10 @@ from deeptutor.services import file_iofrom deeptutor.services.file_io import atomic_write_json, atomic_write_text+def _failing_fsync(_fd: int) -> None:+ raise OSError("simulated disk failure")++def test_atomic_write_json_creates_parent_and_replaces_content(tmp_path: Path) -> None:path = tmp_path / "nested" / "settings.json"@@ -59,3 +65,72 @@ def test_atomic_write_text_creates_parent_and_replaces_content(tmp_path: Path) -assert path.read_text(encoding="utf-8") == "second"assert list(path.parent.iterdir()) == [path]+++def test_atomic_write_json_warns_and_still_writes_when_fsync_fails(+ tmp_path: Path,+ monkeypatch: pytest.MonkeyPatch,+ caplog: pytest.LogCaptureFixture,+) -> None:+ monkeypatch.setattr(file_io.os, "fsync", _failing_fsync)+ monkeypatch.setattr(file_io, "_last_fsync_warning_at", None)+ path = tmp_path / "nested" / "settings.json"++ with caplog.at_level(logging.WARNING, logger="deeptutor.services.file_io"):+ atomic_write_json(path, {"value": 1})++ assert json.loads(path.read_text(encoding="utf-8")) == {"value": 1}+ assert [p.name for p in path.parent.iterdir()] == ["settings.json"]+ warnings = [r for r in caplog.records if r.levelno == logging.WARNING]+ assert len(warnings) == 1+ assert "fsync" in warnings[0].getMessage()+ assert str(path) in warnings[0].getMessage()+++def test_atomic_write_text_warns_and_still_writes_when_fsync_fails(+ tmp_path: Path,+ monkeypatch: pytest.MonkeyPatch,+ caplog: pytest.LogCaptureFixture,+) -> None:+ monkeypatch.setattr(file_io.os, "fsync", _failing_fsync)+ monkeypatch.setattr(file_io, "_last_fsync_warning_at", None)+ path = tmp_path / "nested" / "note.md"++ with caplog.at_level(logging.WARNING, logger="deeptutor.services.file_io"):+ atomic_write_text(path, "content")++ assert path.read_text(encoding="utf-8") == "content"+ assert [p.name for p in path.parent.iterdir()] == ["note.md"]+ warnings = [r for r in caplog.records if r.levelno == logging.WARNING]+ assert len(warnings) == 1+ assert "fsync" in warnings[0].getMessage()+ assert str(path) in warnings[0].getMessage()+++def test_fsync_warning_is_rate_limited_but_recovers_after_interval(+ tmp_path: Path,+ monkeypatch: pytest.MonkeyPatch,+ caplog: pytest.LogCaptureFixture,+) -> None:+ monkeypatch.setattr(file_io.os, "fsync", _failing_fsync)+ monkeypatch.setattr(file_io, "_last_fsync_warning_at", None)+ path = tmp_path / "nested" / "note.md"++ with caplog.at_level(logging.WARNING, logger="deeptutor.services.file_io"):+ for attempt in range(3):+ atomic_write_text(path, f"attempt-{attempt}")++ assert path.read_text(encoding="utf-8") == "attempt-2"+ assert len([r for r in caplog.records if r.levelno == logging.WARNING]) == 1++ monkeypatch.setattr(+ file_io,+ "_last_fsync_warning_at",+ time.monotonic() - file_io._FSYNC_WARNING_INTERVAL_SECONDS - 1.0,+ )++ with caplog.at_level(logging.WARNING, logger="deeptutor.services.file_io"):+ atomic_write_text(path, "after-interval")++ assert path.read_text(encoding="utf-8") == "after-interval"+ assert len([r for r in caplog.records if r.levelno == logging.WARNING]) == 2