DeepTutor: Lifelong Personalized Tutoring. https://deeptutor.info/.
Python ai-agentsai-tutorclawbotcli-tooldeepresearchinteractive-learninglearningmulti-agent-systemsrag
fix: retry transient locks and decode UTF-8 when reading the indexing journal
Mohammad Hijjawib273dc584a0d
diff --git a/deeptutor/knowledge/indexing_run.py b/deeptutor/knowledge/indexing_run.pyindex 43ce47016..d1f5d7d7e 100644--- a/deeptutor/knowledge/indexing_run.py+++ b/deeptutor/knowledge/indexing_run.py@@ -25,7 +25,7 @@ try:except ImportError: # CLI-only installs need no process-monitor dependency.psutil = None-from deeptutor.services.file_io import atomic_write_json+from deeptutor.services.file_io import atomic_write_json, read_text_CURRENT: ContextVar[IndexingRun | None] = ContextVar("indexing_run", default=None)_TERMINAL = {"completed", "partial", "failed", "cancelled", "interrupted"}@@ -85,7 +85,7 @@ def failure_code(exc: BaseException) -> str:def load_run(kb_dir: Path) -> dict[str, Any] | None:path = Path(kb_dir) / ".indexing-run.json"try:- value = json.loads(path.read_text())+ value = json.loads(read_text(path))except FileNotFoundError:return Noneexcept (OSError, ValueError) as exc:@@ -97,7 +97,7 @@ def load_run(kb_dir: Path) -> dict[str, Any] | None:):raise OSError("Indexing journal is invalid; original file preserved.")try:- cancelled = json.loads((Path(kb_dir) / ".indexing-cancel.json").read_text())+ cancelled = json.loads(read_text(Path(kb_dir) / ".indexing-cancel.json"))except FileNotFoundError:cancelled = {}except (OSError, ValueError) as exc:diff --git a/deeptutor/services/file_io.py b/deeptutor/services/file_io.pyindex 9ccc02be5..4ada6ae35 100644--- a/deeptutor/services/file_io.py+++ b/deeptutor/services/file_io.py@@ -9,10 +9,12 @@ from pathlib import Pathimport tempfileimport threadingimport time-from typing import Any+from typing import Any, Callable, TypeVarlogger = logging.getLogger(__name__)+T = TypeVar("T")+# fsync failures weaken the durability promise but do not corrupt the write:# the replacement below still publishes complete content. Warn about them,# rate-limited, so a failing disk fills the log once per interval instead of@@ -45,28 +47,42 @@ def _sync_to_disk(handle: Any, path: Path) -> None:)-def _atomic_replace(src: Path, dst: Path, *, max_retries: int = 5) -> None:- """Replace *dst* with *src*, retrying on transient ``PermissionError``.+def _retry_permission_error(operation: Callable[[], T], *, max_retries: int) -> T:+ """Run *operation*, retrying a transient ``PermissionError`` with backoff.- On Windows the destination file is occasionally locked by another process- (antivirus, indexer, or a concurrent reader), which makes ``Path.replace``+ On Windows a file is occasionally locked by another process (antivirus,+ indexer, or a concurrent reader), which makes opening or replacing itraise ``PermissionError``. A short exponential backoff recovers in most- cases without losing the already-written content.+ cases; a lock that outlasts it is re-raised unchanged."""delay = 0.2- last_err: PermissionError | None = Nonefor attempt in range(max_retries):try:- src.replace(dst)- return- except PermissionError as exc: # pragma: no cover - platform specific- last_err = exc+ return operation()+ except PermissionError:if attempt == max_retries - 1:- break+ raisetime.sleep(delay)delay = min(delay * 2, 1.6)- assert last_err is not None- raise last_err+ raise ValueError("max_retries must be at least 1")+++def _atomic_replace(src: Path, dst: Path, *, max_retries: int = 5) -> None:+ """Replace *dst* with *src*, retrying on transient ``PermissionError``."""+ _retry_permission_error(lambda: src.replace(dst), max_retries=max_retries)+++def read_text(path: Path, *, max_retries: int = 5) -> str:+ """Read a UTF-8 file written by the atomic helpers below.++ Reads use the writers' encoding instead of the locale default, and a+ transient Windows sharing lock gets the same bounded retry as writes.+ ``FileNotFoundError`` and decoding errors propagate immediately.+ """+ path = Path(path)+ return _retry_permission_error(+ lambda: path.read_text(encoding="utf-8"), max_retries=max_retries+ )def atomic_write_json(path: Path, payload: dict[str, Any]) -> None:@@ -114,4 +130,4 @@ def atomic_write_text(path: Path, text: str) -> None:temporary_path.unlink(missing_ok=True)-__all__ = ["atomic_write_json", "atomic_write_text"]+__all__ = ["atomic_write_json", "atomic_write_text", "read_text"]diff --git a/tests/knowledge/test_indexing_run.py b/tests/knowledge/test_indexing_run.pyindex 0ecfe6556..6b1b9874a 100644--- a/tests/knowledge/test_indexing_run.py+++ b/tests/knowledge/test_indexing_run.py@@ -177,3 +177,63 @@ async def test_readiness_is_local_and_cancel_requires_writable_owned_task(tmp_pawith pytest.raises(HTTPException) as error:await knowledge.cancel_indexing_run("kb", "someone-else")assert error.value.status_code == 403+++_READ_TEXT = Path.read_text+++def _lock_reads(monkeypatch, path: Path, failures: int) -> list[Path]:+ """Deny the next *failures* reads of *path* like a Windows sharing lock."""+ from deeptutor.services import file_io++ attempts: list[Path] = []++ def read_text(self, *args, **kwargs):+ if self == path:+ attempts.append(self)+ if len(attempts) <= failures:+ raise PermissionError(13, "Permission denied", str(self))+ return _READ_TEXT(self, *args, **kwargs)++ monkeypatch.setattr(Path, "read_text", read_text)+ monkeypatch.setattr(file_io.time, "sleep", lambda _seconds: None)+ return attempts+++def test_transient_journal_lock_is_retried_and_persistent_lock_fails(tmp_path, monkeypatch):+ journal = tmp_path / ".indexing-run.json"+ cancel = tmp_path / ".indexing-cancel.json"+ with IndexingRun(tmp_path, [], task_id="one") as run:+ attempts = _lock_reads(monkeypatch, journal, failures=2)+ run.check()+ assert len(attempts) == 3++ assert request_cancel(tmp_path, "one")+ attempts = _lock_reads(monkeypatch, cancel, failures=1)+ with pytest.raises(IndexingCancelled):+ run.check()+ assert len(attempts) == 2++ _lock_reads(monkeypatch, journal, failures=100)+ with pytest.raises(OSError, match="unreadable"):+ load_run(tmp_path)+ monkeypatch.undo()+ run.finish("cancelled")+ assert load_run(tmp_path)["state"] == "cancelled"+++def test_journal_is_read_as_utf8_regardless_of_locale(tmp_path, monkeypatch):+ source = tmp_path / "raw" / "ÁLGEBRA 数学.pdf"+ source.parent.mkdir()+ source.write_bytes(b"textbook")++ def read_text(self, encoding=None, errors=None, **kwargs):+ # Simulate a Windows cp1252 locale, the default when no encoding is given.+ return _READ_TEXT(self, encoding=encoding or "cp1252", errors=errors, **kwargs)++ monkeypatch.setattr(Path, "read_text", read_text)+ with IndexingRun(tmp_path, [str(source)], task_id="one") as run:+ run.check()+ run.document(source, "parsed")+ run.finish("failed")+ assert list(visible_run(tmp_path)["documents"]) == ["ÁLGEBRA 数学.pdf"]diff --git a/tests/services/test_file_io.py b/tests/services/test_file_io.pyindex 81e07a174..3a516bbb1 100644--- a/tests/services/test_file_io.py+++ b/tests/services/test_file_io.py@@ -134,3 +134,40 @@ def test_fsync_warning_is_rate_limited_but_recovers_after_interval(assert path.read_text(encoding="utf-8") == "after-interval"assert len([r for r in caplog.records if r.levelno == logging.WARNING]) == 2+++def test_read_text_retries_transient_permission_error_then_gives_up(+ tmp_path: Path, monkeypatch: pytest.MonkeyPatch+) -> None:+ path = tmp_path / "state.json"+ atomic_write_json(path, {"title": "Résumé 数学"})+ original = Path.read_text+ attempts: list[str | None] = []+ sleeps: list[float] = []++ def locked_read(self: Path, encoding: str | None = None, errors: str | None = None) -> str:+ attempts.append(encoding)+ if len(attempts) <= failures:+ raise PermissionError(13, "Permission denied", str(self))+ return original(self, encoding=encoding, errors=errors)++ monkeypatch.setattr(Path, "read_text", locked_read)+ monkeypatch.setattr(file_io.time, "sleep", sleeps.append)++ failures = 2+ assert json.loads(file_io.read_text(path)) == {"title": "Résumé 数学"}+ assert attempts == ["utf-8"] * 3+ assert sleeps == [0.2, 0.4]++ attempts.clear()+ sleeps.clear()+ failures = 100+ with pytest.raises(PermissionError):+ file_io.read_text(path)+ assert len(attempts) == 5+ assert sleeps == [0.2, 0.4, 0.8, 1.6]+++def test_read_text_does_not_retry_missing_file(tmp_path: Path) -> None:+ with pytest.raises(FileNotFoundError):+ file_io.read_text(tmp_path / "missing.json")