Skip to content

The agent that grows with you

fix(cache): a file that stats but cannot be read keys a real read, never raises from the digest

Teknium82d5c0ae8106
diff --git a/hermes_cli/config_read_errors.py b/hermes_cli/config_read_errors.py
index 11666bb4a48..5bf73ed12ef 100644
--- a/hermes_cli/config_read_errors.py
+++ b/hermes_cli/config_read_errors.py
@@ -101,7 +101,7 @@ def _warn_config_parse_failure(
def _failure_version(path: Path) -> tuple:
"""What a parse-failure record is matched on: ``file_cache_version`` hashed even once settled,
because the record is compared long after it was taken and a same-tick same-size fix must clear
- it. An unreadable file (the OSError failures) keeps its stat signature and a ``None`` digest."""
+ it. An unreadable file (the OSError failures) gets its stat signature and the error's name."""
from utils import file_cache_version
try:
return file_cache_version(path, always_digest=True)
diff --git a/tests/hermes_cli/test_config_same_tick_rewrite.py b/tests/hermes_cli/test_config_same_tick_rewrite.py
index d9b73e0050e..04d6ade0aa3 100644
--- a/tests/hermes_cli/test_config_same_tick_rewrite.py
+++ b/tests/hermes_cli/test_config_same_tick_rewrite.py
@@ -32,14 +32,52 @@ def _readers():
}
+class _Clock:
+ """``utils``' view of the ``time`` module with ``time_ns`` under the test's control."""
+
+ def __init__(self, now):
+ self.now = now
+
+ def time_ns(self):
+ return self.now
+
+ def __getattr__(self, name):
+ return getattr(time, name)
+
+
@pytest.fixture
def coarse_clock(monkeypatch):
- """Every stat signature reports one frozen mtime/ctime tick, like a coarse filesystem clock."""
+ """Every stat signature reports one frozen mtime/ctime tick, like a coarse filesystem clock; the
+ returned clock is what ``utils`` reads as now, so a test can move past the racy window."""
import utils
real, tick = utils.file_signature, time.time_ns()
monkeypatch.setattr(utils, 'file_signature', lambda st: (tick, *real(st)[1:3], tick))
+ clock = _Clock(tick)
+ monkeypatch.setattr(utils, 'time', clock)
+ return clock
+
+
+def digest_opens(monkeypatch):
+ """Paths opened through ``utils`` (the cache-key content digest, and some parsers)."""
+ import builtins
+ import utils
+ opened = []
+
+ def _open(file, *args, **kwargs):
+ opened.append(os.fspath(file))
+ return builtins.open(file, *args, **kwargs)
+
+ monkeypatch.setattr(utils, 'open', _open, raising=False)
+ return opened
+def frozen_now(monkeypatch):
+ """``utils`` reads now as the test's start, so every file the test writes stays racy (fresh)."""
+ import utils
+ monkeypatch.setattr(utils, 'time', _Clock(time.time_ns()))
+
def test_same_size_rewrite_in_one_tick_reaches_every_config_reader(coarse_clock):
from hermes_cli.config import get_config_path
path = get_config_path()
@@ -53,15 +91,56 @@ def test_same_size_rewrite_in_one_tick_reaches_every_config_reader(coarse_clock)
readers, 'http://127.0.0.1:5/v1')
-def test_an_unchanged_file_stays_one_parse_while_fresh_and_once_settled(coarse_clock):
+def test_an_unchanged_file_stays_one_parse_while_fresh_and_once_settled(coarse_clock, digest_opens):
+ import utils
from hermes_cli.config import get_config_path
path = get_config_path()
path.parent.mkdir(parents=True, exist_ok=True)
readonly = {k: v for k, v in _readers().items() if k in ('load_config_readonly', 'read_config_doc_readonly')}
_write(path, 3)
- for read in readonly.values(): # inside the racy window: the content digest matches, still a hit
+ for read in readonly.values():
+ read()
+ digest_opens.clear()
+ for read in readonly.values(): # inside the racy window: hashed for the key, the digest matches, a hit
assert read() is read()
- settled = time.time_ns() - 10_000_000_000
- os.utime(path, ns=(settled, settled))
+ assert str(path) in digest_opens
+ digest_opens.clear()
+ coarse_clock.now += utils.RACY_MTIME_WINDOW_NS + 1
for read in readonly.values():
+ read() # settled: one re-parse onto the stat-only key
+ digest_opens.clear()
+ for read in readonly.values(): # then hits that never open the file
assert read() is read()
+ assert str(path) not in digest_opens
+
+
[email protected]("linux", "macos")
[email protected](hasattr(os, 'geteuid') and os.geteuid() == 0, reason='root reads a mode-000 file')
+def test_a_fresh_config_turned_unreadable_keeps_the_last_good_settings(frozen_now, monkeypatch, tmp_path, caplog):
+ """A fresh file that stats but cannot be read reaches each loader's read-error recovery: the
+ content digest in the cache key must not raise where the stat-only key never opened the file."""
+ from hermes_cli.config import get_config_path, load_config
+ from hermes_cli.config_effective import load_user_config_effective
+ from hermes_cli.managed_scope import invalidate_managed_cache, load_managed_config
+ path = get_config_path()
+ path.parent.mkdir(parents=True, exist_ok=True)
+ managed = tmp_path / 'managed'
+ managed.mkdir()
+ monkeypatch.setenv('HERMES_MANAGED_DIR', str(managed))
+ invalidate_managed_cache()
+ (managed / 'config.yaml').write_text(json.dumps({'agent': {'max_turns': 7}}))
+ _write(path, 3)
+ assert load_config()['model']['base_url'] == 'http://127.0.0.1:3/v1'
+ assert load_user_config_effective(path)['model']['base_url'] == 'http://127.0.0.1:3/v1'
+ assert load_managed_config() == {'agent': {'max_turns': 7}}
+ path.chmod(0)
+ (managed / 'config.yaml').chmod(0)
+ try:
+ assert load_config()['model']['base_url'] == 'http://127.0.0.1:3/v1'
+ assert load_user_config_effective(path)['model']['base_url'] == 'http://127.0.0.1:3/v1'
+ assert load_managed_config() == {}
+ assert 'IGNORING this managed file' in caplog.text
+ finally:
+ path.chmod(0o600)
+ (managed / 'config.yaml').chmod(0o600)
+ invalidate_managed_cache()
diff --git a/utils.py b/utils.py
index 0b7d4d66522..8d3193114fa 100644
--- a/utils.py
+++ b/utils.py
@@ -76,17 +76,29 @@ def _frozen_config_worker(path: str | Path) -> bool:
return policy is not None and policy.worker_config_snapshot() is not None
+def _file_digest(path: str | Path) -> str:
+ """:func:`content_digest` of the file's bytes. A file that stats but cannot be read here
+ (permission, removed since the stat, a directory) gets ``<ErrorName>``, which no hex digest
+ matches: the key differs from any cached one, so the caller's own read runs and meets the error
+ in its normal recovery, as it did when the key was stat-only and never opened the file."""
+ try:
+ with open(path, "rb") as f:
+ return content_digest(f.read())
+ except OSError as exc:
+ return f"<{type(exc).__name__}>"
+
+
def racy_content_digest(path: str | Path, sig: tuple[int, int, int, int]) -> str:
"""``""`` once the :func:`file_signature` ``sig`` has settled; while it is racy, a digest of the
- file's bytes. A cache keyed on a stat signature appends it, because two same-size in-place writes
- in one timestamp tick share mtime, ctime, size and inode (kernels without multigrain timestamps).
- Racy is judged on the later of mtime and ctime: userland can restore an older mtime after a write
- (``cp -p``, ``rsync --inplace -t``), never ctime. Read before the parse it guards: a write in
- between then costs one re-parse instead of pinning the older content."""
+ file's bytes (:func:`_file_digest`, so it never raises). A cache keyed on a stat signature appends
+ it, because two same-size in-place writes in one timestamp tick share mtime, ctime, size and inode
+ (kernels without multigrain timestamps). Racy is judged on the later of mtime and ctime: userland
+ can restore an older mtime after a write (``cp -p``, ``rsync --inplace -t``), never ctime. Read
+ before the parse it guards: a write in between then costs one re-parse instead of pinning the
+ older content."""
if not mtime_is_racy(max(sig[0], sig[3])) or _frozen_config_worker(path):
return ""
- with open(path, "rb") as f:
- return content_digest(f.read())
+ return _file_digest(path)
def file_cache_version(path: str | Path, st: os.stat_result | None = None, *,
@@ -99,18 +111,12 @@ def file_cache_version(path: str | Path, st: os.stat_result | None = None, *,
change detector whose false "changed" restarts something, or a "still the broken file" record),
where the racy digest turning empty would read as a change. A worker with a frozen config
snapshot never hashes a ``config.yaml`` (the key is stat-only there, as before). Raises
- ``OSError`` like ``os.stat``; given the caller's ``st`` it does not: a file gone or unreadable
- since that stat gets the error's name as its digest, which no readable version matches."""
+ ``OSError`` only from the stat (none given ``st``): a file that cannot be read gets the error's
+ name as its digest (:func:`_file_digest`)."""
sig = file_signature(Path(path).stat() if st is None else st)
- try:
- if always_digest and not _frozen_config_worker(path):
- with open(path, "rb") as f:
- return (*sig, content_digest(f.read()))
- return (*sig, racy_content_digest(path, sig))
- except OSError as exc:
- if st is None:
- raise
- return (*sig, type(exc).__name__)
+ if always_digest and not _frozen_config_worker(path):
+ return (*sig, _file_digest(path))
+ return (*sig, racy_content_digest(path, sig))
def _preserve_file_mode(path: Path) -> int | None: