Skip to content

DeepTutor: Lifelong Personalized Tutoring. https://deeptutor.info/.

fix(services): surface fsync failures in atomic writers as rate-limited warnings (#1751)

evan188199-tech409fa3c4e14c
diff --git a/deeptutor/services/file_io.py b/deeptutor/services/file_io.py
index bdf64c230..9ccc02be5 100644
--- a/deeptutor/services/file_io.py
+++ b/deeptutor/services/file_io.py
@@ -3,12 +3,47 @@
from __future__ import annotations
import json
+import logging
import os
from pathlib import Path
import tempfile
+import threading
import time
from typing import Any
+logger = logging.getLogger(__name__)
+
+# fsync failures weaken the durability promise but do not corrupt the write:
+# the replacement below still publishes complete content. Warn about them,
+# rate-limited, so a failing disk fills the log once per interval instead of
+# once per write.
+_FSYNC_WARNING_INTERVAL_SECONDS = 60.0
+_last_fsync_warning_at: float | None = None
+_fsync_warning_lock = threading.Lock()
+
+
+def _sync_to_disk(handle: Any, path: Path) -> None:
+ """fsync *handle*, surfacing a failure as a rate-limited warning."""
+ global _last_fsync_warning_at
+ try:
+ os.fsync(handle.fileno())
+ except OSError as exc:
+ now = time.monotonic()
+ with _fsync_warning_lock:
+ should_warn = (
+ _last_fsync_warning_at is None
+ or now - _last_fsync_warning_at >= _FSYNC_WARNING_INTERVAL_SECONDS
+ )
+ if should_warn:
+ _last_fsync_warning_at = now
+ if should_warn:
+ logger.warning(
+ "fsync failed while writing %s; the write succeeded but is not"
+ " guaranteed durable on disk (%r)",
+ path,
+ exc,
+ )
+
def _atomic_replace(src: Path, dst: Path, *, max_retries: int = 5) -> None:
"""Replace *dst* with *src*, retrying on transient ``PermissionError``.
@@ -50,10 +85,7 @@ def atomic_write_json(path: Path, payload: dict[str, Any]) -> None:
json.dump(payload, handle, indent=2, ensure_ascii=False)
handle.write("\n")
handle.flush()
- try:
- os.fsync(handle.fileno())
- except OSError:
- pass
+ _sync_to_disk(handle, path)
_atomic_replace(temporary_path, path)
finally:
if temporary_path is not None:
@@ -75,10 +107,7 @@ def atomic_write_text(path: Path, text: str) -> None:
temporary_path = Path(handle.name)
handle.write(text)
handle.flush()
- try:
- os.fsync(handle.fileno())
- except OSError:
- pass
+ _sync_to_disk(handle, path)
_atomic_replace(temporary_path, path)
finally:
if temporary_path is not None:
diff --git a/tests/services/test_file_io.py b/tests/services/test_file_io.py
index 2c4107830..81e07a174 100644
--- a/tests/services/test_file_io.py
+++ b/tests/services/test_file_io.py
@@ -1,7 +1,9 @@
from __future__ import annotations
import json
+import logging
from pathlib import Path
+import time
import pytest
@@ -9,6 +11,10 @@ from deeptutor.services import file_io
from deeptutor.services.file_io import atomic_write_json, atomic_write_text
+def _failing_fsync(_fd: int) -> None:
+ raise OSError("simulated disk failure")
+
+
def test_atomic_write_json_creates_parent_and_replaces_content(tmp_path: Path) -> None:
path = tmp_path / "nested" / "settings.json"
@@ -59,3 +65,72 @@ def test_atomic_write_text_creates_parent_and_replaces_content(tmp_path: Path) -
assert path.read_text(encoding="utf-8") == "second"
assert list(path.parent.iterdir()) == [path]
+
+
+def test_atomic_write_json_warns_and_still_writes_when_fsync_fails(
+ tmp_path: Path,
+ monkeypatch: pytest.MonkeyPatch,
+ caplog: pytest.LogCaptureFixture,
+) -> None:
+ monkeypatch.setattr(file_io.os, "fsync", _failing_fsync)
+ monkeypatch.setattr(file_io, "_last_fsync_warning_at", None)
+ path = tmp_path / "nested" / "settings.json"
+
+ with caplog.at_level(logging.WARNING, logger="deeptutor.services.file_io"):
+ atomic_write_json(path, {"value": 1})
+
+ assert json.loads(path.read_text(encoding="utf-8")) == {"value": 1}
+ assert [p.name for p in path.parent.iterdir()] == ["settings.json"]
+ warnings = [r for r in caplog.records if r.levelno == logging.WARNING]
+ assert len(warnings) == 1
+ assert "fsync" in warnings[0].getMessage()
+ assert str(path) in warnings[0].getMessage()
+
+
+def test_atomic_write_text_warns_and_still_writes_when_fsync_fails(
+ tmp_path: Path,
+ monkeypatch: pytest.MonkeyPatch,
+ caplog: pytest.LogCaptureFixture,
+) -> None:
+ monkeypatch.setattr(file_io.os, "fsync", _failing_fsync)
+ monkeypatch.setattr(file_io, "_last_fsync_warning_at", None)
+ path = tmp_path / "nested" / "note.md"
+
+ with caplog.at_level(logging.WARNING, logger="deeptutor.services.file_io"):
+ atomic_write_text(path, "content")
+
+ assert path.read_text(encoding="utf-8") == "content"
+ assert [p.name for p in path.parent.iterdir()] == ["note.md"]
+ warnings = [r for r in caplog.records if r.levelno == logging.WARNING]
+ assert len(warnings) == 1
+ assert "fsync" in warnings[0].getMessage()
+ assert str(path) in warnings[0].getMessage()
+
+
+def test_fsync_warning_is_rate_limited_but_recovers_after_interval(
+ tmp_path: Path,
+ monkeypatch: pytest.MonkeyPatch,
+ caplog: pytest.LogCaptureFixture,
+) -> None:
+ monkeypatch.setattr(file_io.os, "fsync", _failing_fsync)
+ monkeypatch.setattr(file_io, "_last_fsync_warning_at", None)
+ path = tmp_path / "nested" / "note.md"
+
+ with caplog.at_level(logging.WARNING, logger="deeptutor.services.file_io"):
+ for attempt in range(3):
+ atomic_write_text(path, f"attempt-{attempt}")
+
+ assert path.read_text(encoding="utf-8") == "attempt-2"
+ assert len([r for r in caplog.records if r.levelno == logging.WARNING]) == 1
+
+ monkeypatch.setattr(
+ file_io,
+ "_last_fsync_warning_at",
+ time.monotonic() - file_io._FSYNC_WARNING_INTERVAL_SECONDS - 1.0,
+ )
+
+ with caplog.at_level(logging.WARNING, logger="deeptutor.services.file_io"):
+ atomic_write_text(path, "after-interval")
+
+ assert path.read_text(encoding="utf-8") == "after-interval"
+ assert len([r for r in caplog.records if r.levelno == logging.WARNING]) == 2