Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

fix(security): lstat before narrowing sqlite files (#4527)

BBP2cf64ef0cc91
diff --git a/headroom/fileperms.py b/headroom/fileperms.py
index 668eb40ef..d3a11379b 100644
--- a/headroom/fileperms.py
+++ b/headroom/fileperms.py
@@ -230,8 +230,8 @@ def ensure_private_file(path: str | os.PathLike[str], *, what: str = "file") ->
opened world-readable, so a failure to create or narrow the file privately
raises :class:`PermissionError` rather than proceeding to open a wide
file. The existing-file path is symlink-race resistant — ``O_EXCL`` on the
- create refuses to reuse a planted file or symlink; an existing file is
- re-opened with ``O_NOFOLLOW`` and narrowed through that descriptor
+ create refuses to reuse a planted file or symlink; an existing file not
+ already 0600 is opened with ``O_NOFOLLOW`` and narrowed through it
(``fstat`` to confirm a regular file, ``fchmod`` to set the mode) rather
than by re-resolving the path, which closes the check-then-chmod TOCTOU a
``chmod(path)`` would leave open. On Windows POSIX mode bits and
@@ -249,15 +249,36 @@ def ensure_private_file(path: str | os.PathLike[str], *, what: str = "file") ->
except FileExistsError:
pass
+ _narrow_existing(path, what=what, missing_ok=False)
+
+
+def _narrow_existing(path: str | os.PathLike[str], *, what: str, missing_ok: bool) -> None:
+ """Narrow an existing file to 0600 through an ``O_NOFOLLOW`` descriptor.
+
+ A regular file already 0600 and owned by the caller (any owner as root) is
+ only ``lstat``-ed: closing any descriptor on a sqlite file drops the
+ process's ``fcntl`` locks on it (sqlite.org/howtocorrupt.html, 2.2).
+ Non-regular files are refused; with *missing_ok*, a missing file is skipped.
+ """
if not (hasattr(os, "O_NOFOLLOW") and hasattr(os, "fchmod")):
return
+ fd = -1
try:
- fd = os.open(path, os.O_RDONLY | os.O_NOFOLLOW)
+ st = os.lstat(path)
+ if stat.S_ISREG(st.st_mode):
+ euid = os.geteuid() # type: ignore[attr-defined]
+ if stat.S_IMODE(st.st_mode) == OWNER_ONLY_MODE and (st.st_uid == euid or euid == 0):
+ return
+ fd = os.open(path, os.O_RDONLY | os.O_NOFOLLOW)
except OSError as exc:
+ if missing_ok and isinstance(exc, FileNotFoundError):
+ return
raise PermissionError(
f"refusing to open {what} at {path}: not a regular file (symlink or open error: {exc})"
) from exc
+ if fd < 0:
+ raise PermissionError(f"refusing to open {what} at {path}: not a regular file")
try:
if not stat.S_ISREG(os.fstat(fd).st_mode):
raise PermissionError(f"refusing to open {what} at {path}: not a regular file")
@@ -346,9 +367,7 @@ def connect_private_sqlite(
# them. Missing ones are left alone: sqlite creates them with the
# main file's mode, which is now 0600.
for suffix in _SQLITE_SIDECAR_SUFFIXES:
- sidecar = file_path + suffix
- if os.path.lexists(sidecar):
- ensure_private_file(sidecar, what=f"{what} {suffix[1:]} file")
+ _narrow_existing(file_path + suffix, what=f"{what} {suffix[1:]} file", missing_ok=True)
# ``**connect_kwargs: Any`` makes mypy type the call as ``Any``; the
# annotation pins it back to the real return type.
conn: sqlite3.Connection = sqlite3.connect(os.fspath(path), **connect_kwargs)
diff --git a/tests/test_private_state_files.py b/tests/test_private_state_files.py
index 995675ad9..7b0fb7bf4 100644
--- a/tests/test_private_state_files.py
+++ b/tests/test_private_state_files.py
@@ -14,6 +14,7 @@ result is known to be umask-independent.
from __future__ import annotations
import os
+import signal
import sqlite3
import stat
import subprocess
@@ -41,6 +42,55 @@ def _permissive_umask():
os.umask(old)
+def fail_instead_of_hang():
+ def on_alarm(signum, frame):
+ raise AssertionError("blocked opening a FIFO")
+
+ old = signal.signal(signal.SIGALRM, on_alarm)
+ signal.alarm(5)
+ try:
+ yield
+ finally:
+ signal.alarm(0)
+ signal.signal(signal.SIGALRM, old)
+
+
+def _record_opens(monkeypatch, target) -> list:
+ real_open = os.open
+ opened = []
+
+ def record(path, flags, *args, **kwargs):
+ if os.fspath(path) == str(target) and not flags & os.O_CREAT:
+ opened.append(flags)
+ return real_open(path, flags, *args, **kwargs)
+
+ monkeypatch.setattr(os, "open", record)
+ return opened
+
+
+def _report_lstat(monkeypatch, target, *, mode=None, uid=None) -> None:
+ """Make ``os.lstat(target)`` report a different type/mode or owner.
+
+ The real file is untouched; only what ``lstat`` says about it changes, which
+ stands in for a swap or a foreign owner that cannot be arranged without root.
+ """
+ real_lstat = os.lstat
+
+ def lstat(path, *args, **kwargs):
+ st = real_lstat(path, *args, **kwargs)
+ if os.fspath(path) != str(target):
+ return st
+ fields = list(st)
+ if mode is not None:
+ fields[0] = mode
+ if uid is not None:
+ fields[4] = uid
+ return os.stat_result(fields)
+
+ monkeypatch.setattr(os, "lstat", lstat)
+
+
# ---- primitives -------------------------------------------------------------
@@ -86,6 +136,85 @@ class TestEnsurePrivateFile:
with pytest.raises(PermissionError):
fileperms.ensure_private_file(p)
+ @pytest.mark.parametrize("mode", [0o400, 0o700, 0o4600])
+ def test_non_0600_mode_is_narrowed_once_then_skipped(self, tmp_path, monkeypatch, mode):
+ p = tmp_path / "store.db"
+ p.write_text("x")
+ p.chmod(mode)
+ assert _mode(p) == mode
+ opened = _record_opens(monkeypatch, p)
+ fileperms.ensure_private_file(p)
+ assert len(opened) == 1
+ assert _mode(p) == 0o600
+ fileperms.ensure_private_file(p)
+ assert len(opened) == 1
+ assert p.read_text() == "x"
+
+ def test_0600_file_owned_by_the_caller_is_not_opened(self, tmp_path, monkeypatch):
+ p = tmp_path / "store.db"
+ p.write_text("x")
+ p.chmod(0o600)
+ opened = _record_opens(monkeypatch, p)
+ fileperms.ensure_private_file(p)
+ assert opened == []
+
+ def test_0600_file_reported_foreign_to_a_non_root_caller_is_opened_once(
+ self, tmp_path, monkeypatch
+ ):
+ # The file is really ours, so the fchmod that follows the open succeeds
+ # and leaves it 0600. A truly foreign file would fail that fchmod with
+ # EPERM and surface as PermissionError, which needs root to arrange.
+ p = tmp_path / "store.db"
+ p.write_text("x")
+ p.chmod(0o600)
+ real_uid = os.geteuid()
+ non_root_uid = real_uid or 1000
+ monkeypatch.setattr(os, "geteuid", lambda: non_root_uid)
+ _report_lstat(monkeypatch, p, uid=non_root_uid + 1)
+ opened = _record_opens(monkeypatch, p)
+ fileperms.ensure_private_file(p)
+ assert len(opened) == 1
+ assert _mode(p) == 0o600
+
+ def test_0600_file_of_another_owner_is_not_opened_when_running_as_root(
+ self, tmp_path, monkeypatch
+ ):
+ # geteuid is patched to 0 and lstat reports owner uid 1, while the file
+ # really belongs to the test user: root on a file of another owner.
+ p = tmp_path / "store.db"
+ p.write_text("x")
+ p.chmod(0o600)
+ monkeypatch.setattr(os, "geteuid", lambda: 0)
+ _report_lstat(monkeypatch, p, uid=1)
+ opened = _record_opens(monkeypatch, p)
+ fileperms.ensure_private_file(p)
+ assert opened == []
+
+ def test_unreadable_wide_file_is_refused_with_context(self, tmp_path):
+ if os.geteuid() == 0:
+ pytest.skip("root can open any file")
+ p = tmp_path / "store.db"
+ p.write_text("x")
+ p.chmod(0o044)
+ with pytest.raises(PermissionError, match="refusing to open memory store") as info:
+ fileperms.ensure_private_file(p, what="memory store")
+ assert isinstance(info.value.__cause__, PermissionError)
+ assert _mode(p) == 0o044
+
+ def test_symlink_swapped_in_after_lstat_is_refused_and_target_untouched(
+ self, tmp_path, monkeypatch
+ ):
+ target = tmp_path / "victim"
+ target.write_text("SECRET")
+ link = tmp_path / "store.db"
+ link.symlink_to(target)
+ _report_lstat(monkeypatch, link, mode=stat.S_IFREG | 0o644)
+ with pytest.raises(PermissionError, match="refusing to open memory store") as info:
+ fileperms.ensure_private_file(link, what="memory store")
+ assert isinstance(info.value.__cause__, OSError)
+ assert _mode(target) == 0o644
+ assert target.read_text() == "SECRET"
+
class TestConnectPrivateSqlite:
def test_creates_private_db_and_sidecars(self, tmp_path):
@@ -210,6 +339,29 @@ class TestConnectPrivateSqlite:
]
conn.close()
+ @pytest.mark.parametrize("suffix", ["-wal", "-shm", "-journal"])
+ def test_refuses_dangling_symlink_sidecar(self, tmp_path, suffix):
+ db = tmp_path / "m.db"
+ sqlite3.connect(db).close()
+ (tmp_path / f"m.db{suffix}").symlink_to(tmp_path / "nowhere")
+ with pytest.raises(PermissionError, match="not a regular file"):
+ fileperms.connect_private_sqlite(db)
+ assert not (tmp_path / "nowhere").exists()
+
+ def test_0400_database_is_writable_after_connecting(self, tmp_path):
+ db = tmp_path / "m.db"
+ seed = sqlite3.connect(db)
+ seed.execute("CREATE TABLE t(x)")
+ seed.commit()
+ seed.close()
+ db.chmod(0o400)
+ conn = fileperms.connect_private_sqlite(db)
+ assert _mode(db) == 0o600
+ conn.execute("INSERT INTO t VALUES (1)")
+ conn.commit()
+ assert conn.execute("SELECT x FROM t").fetchall() == [(1,)]
+ conn.close()
+
def test_missing_sidecars_are_not_precreated(self, tmp_path):
db = tmp_path / "m.db"
fileperms.connect_private_sqlite(db).close()
@@ -245,6 +397,127 @@ class TestConnectPrivateSqlite:
with pytest.raises(PermissionError):
fileperms.connect_private_sqlite(link)
+ @pytest.mark.parametrize("suffix", ["-wal", "-shm", "-journal"])
+ def test_sidecar_deleted_by_sqlite_before_narrowing_is_skipped(
+ self, tmp_path, monkeypatch, suffix
+ ):
+ # sqlite unlinks -journal after every transaction and -wal/-shm when the
+ # last connection closes, so a sidecar seen a moment ago can be gone by
+ # the time it is opened. That is not a reason to refuse the store, and
+ # the helper must not recreate it.
+ db = tmp_path / "m.db"
+ fileperms.connect_private_sqlite(db).close()
+ sidecar = tmp_path / f"m.db{suffix}"
+ sidecar.write_bytes(b"leftover")
+ assert _mode(sidecar) == 0o644 # precondition under the 022 umask
+ real_open = os.open
+
+ def open_after_sqlite_deletes_it(path, flags, *args, **kwargs):
+ if os.fspath(path) == str(sidecar) and not flags & os.O_CREAT:
+ sidecar.unlink(missing_ok=True)
+ return real_open(path, flags, *args, **kwargs)
+
+ monkeypatch.setattr(os, "open", open_after_sqlite_deletes_it)
+ fileperms.connect_private_sqlite(db).close()
+ assert sorted(p.name for p in tmp_path.iterdir()) == ["m.db"]
+
+ @pytest.mark.parametrize("journal_mode", ["delete", "wal"])
+ def test_connecting_keeps_other_connections_locks(self, tmp_path, journal_mode):
+ # POSIX close() of any descriptor on a file releases every fcntl lock the
+ # process holds on it, so narrowing must not open and close a file that
+ # is already private. If it does, a second process can take the write
+ # lock while this one is mid-transaction and the database is corrupted.
+ db = tmp_path / "m.db"
+ holder = fileperms.connect_private_sqlite(db, isolation_level=None)
+ holder.execute(f"PRAGMA journal_mode={journal_mode}")
+ holder.execute("CREATE TABLE t(x)")
+ holder.execute("BEGIN IMMEDIATE")
+ holder.execute("INSERT INTO t VALUES (1)")
+
+ fileperms.connect_private_sqlite(db).close()
+
+ script = (
+ "import sqlite3, sys\n"
+ "c = sqlite3.connect(sys.argv[1], timeout=0, isolation_level=None)\n"
+ "try:\n"
+ " c.execute('BEGIN IMMEDIATE')\n"
+ " print('acquired')\n"
+ "except sqlite3.OperationalError as exc:\n"
+ " print(exc)\n"
+ )
+ result = subprocess.run(
+ [sys.executable, "-c", script, str(db)],
+ check=True,
+ capture_output=True,
+ text=True,
+ timeout=30,
+ )
+ assert result.stdout.strip() == "database is locked"
+ holder.execute("COMMIT")
+ assert holder.execute("PRAGMA integrity_check").fetchall() == [("ok",)]
+ holder.close()
+
+ @pytest.mark.parametrize("suffix", ["", "-wal", "-shm", "-journal"])
+ def test_owner_only_file_is_not_opened(self, tmp_path, monkeypatch, suffix):
+ db = tmp_path / "m.db"
+ fileperms.connect_private_sqlite(db).close()
+ target = tmp_path / f"m.db{suffix}"
+ if suffix:
+ target.write_bytes(b"leftover")
+ target.chmod(0o600)
+ assert _mode(target) == 0o600
+ real_open = os.open
+ opened = []
+
+ def record_opened_fds(path, flags, *args, **kwargs):
+ fd = real_open(path, flags, *args, **kwargs)
+ if os.fspath(path) == str(target):
+ opened.append(fd)
+ return fd
+
+ monkeypatch.setattr(os, "open", record_opened_fds)
+ fileperms.connect_private_sqlite(db).close()
+ assert opened == []
+
+ @pytest.mark.parametrize("suffix", ["", "-wal", "-shm", "-journal"])
+ @pytest.mark.parametrize("kind", ["symlink", "fifo"])
+ def test_refuses_non_regular_file(
+ self, tmp_path, monkeypatch, fail_instead_of_hang, kind, suffix
+ ):
+ db = tmp_path / "m.db"
+ if suffix:
+ fileperms.connect_private_sqlite(db).close()
+ bad = tmp_path / f"m.db{suffix}"
+ victim = tmp_path / "victim"
+ victim.write_text("SECRET")
+ if kind == "symlink":
+ bad.symlink_to(victim)
+ else:
+ os.mkfifo(bad)
+ # A FIFO with no writer blocks an open for reading, so it has to be
+ # refused before anything opens it; the alarm fails the test otherwise.
+ opened = _record_opens(monkeypatch, bad)
+ with pytest.raises(PermissionError, match="not a regular file") as info:
+ fileperms.connect_private_sqlite(db, what="memory store")
+ assert "refusing to open memory store" in str(info.value)
+ assert str(bad) in str(info.value)
+ assert opened == []
+ assert _mode(victim) == 0o644
+ assert victim.read_text() == "SECRET"
+
+ def test_wide_database_is_narrowed(self, tmp_path):
+ db = tmp_path / "m.db"
+ seed = sqlite3.connect(db)
+ seed.execute("CREATE TABLE t(x)")
+ seed.execute("INSERT INTO t VALUES ('kept')")
+ seed.commit()
+ seed.close()
+ assert _mode(db) == 0o644 # precondition under the 022 umask
+ conn = fileperms.connect_private_sqlite(db)
+ assert _mode(db) == 0o600
+ assert conn.execute("SELECT x FROM t").fetchall() == [("kept",)]
+ conn.close()
+
class TestPrivateDir:
def test_new_and_existing_dirs_are_0700(self, tmp_path):