Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
fix(security): create credential files private instead of narrowing after write (#3215)
Abhay Singh9b8cae844808
diff --git a/headroom/cache/backends/sqlite.py b/headroom/cache/backends/sqlite.pyindex ab5a35bed..764b138ee 100644--- a/headroom/cache/backends/sqlite.py+++ b/headroom/cache/backends/sqlite.py@@ -22,6 +22,7 @@ import jsonimport loggingimport osimport sqlite3+import statimport threadingimport timefrom dataclasses import asdict, fields@@ -83,7 +84,74 @@ class SQLiteBackend:self._last_purge = 0.0self._conn = self._open()+ @staticmethod+ def _ensure_private(path: Path) -> None:+ """Make the database file 0600 *before* sqlite opens it, failing CLOSED.++ The originals stored here can contain sensitive tool output (file+ contents, command output). ``mkdir`` created the parent at the umask+ default (typically world-traversable ``0o755``), and sqlite would+ otherwise create the db file itself at the umask default too — leaving it+ world/group readable. This creates the file with an explicit ``0o600``+ mode before ``sqlite3.connect`` runs (a *new* db is private from birth;+ sqlite treats the empty file as a fresh database), or narrows a+ pre-existing db.++ A store of raw tool output must not be opened world-readable, so a+ failure to create or narrow it privately **raises** rather than silently+ proceeding to open a wide file.++ The existing-file path is symlink-race resistant. ``O_EXCL`` on the+ create refuses to reuse an attacker-planted file or symlink. When the+ file already exists it is re-opened with ``O_NOFOLLOW`` and narrowed+ through that descriptor (``fstat`` to confirm a regular file, ``fchmod``+ to set the mode) rather than by re-resolving the path: a symlink is+ refused at the ``open`` syscall, and operating on the verified+ descriptor closes the check-then-chmod TOCTOU that a ``chmod(path)``+ (which follows symlinks) would leave open. On Windows POSIX permission+ bits and ``O_NOFOLLOW`` do not apply, so there is nothing to narrow.+ """+ # Fast path: create the file ourselves. O_EXCL (plus O_NOFOLLOW where+ # available) refuses to reuse an existing file or follow a planted+ # symlink, so a brand-new db is private from birth at 0o600.+ create_flags = os.O_CREAT | os.O_EXCL | os.O_WRONLY+ if hasattr(os, "O_NOFOLLOW"):+ create_flags |= os.O_NOFOLLOW+ try:+ os.close(os.open(path, create_flags, 0o600))+ return+ except FileExistsError:+ pass++ # The file already exists. Without POSIX no-follow/fd primitives+ # (Windows) permission bits do not apply and there is no symlink-race+ # hardening to perform; sqlite opens by path as before.+ if not (hasattr(os, "O_NOFOLLOW") and hasattr(os, "fchmod")):+ return++ # Narrow the existing file, but only after proving — through a single+ # no-follow descriptor — that it is a real regular file. A symlink is+ # refused by O_NOFOLLOW (fail closed), and chmod-ing the descriptor+ # rather than the path means an attacker cannot swap the inode we+ # verified between the check and the narrow.+ try:+ fd = os.open(path, os.O_RDONLY | os.O_NOFOLLOW)+ except OSError as exc:+ raise PermissionError(+ f"refusing to open CCR store at {path}: not a regular file "+ f"(symlink or open error: {exc})"+ ) from exc+ try:+ if not stat.S_ISREG(os.fstat(fd).st_mode):+ raise PermissionError(f"refusing to open CCR store at {path}: not a regular file")+ # os.fchmod is POSIX-only (guarded by the hasattr check above); the+ # ignore keeps type-checking clean on Windows where it is absent.+ os.fchmod(fd, 0o600) # type: ignore[attr-defined]+ finally:+ os.close(fd)+def _open(self) -> sqlite3.Connection:+ self._ensure_private(self._path)conn = sqlite3.connect(self._path, check_same_thread=False)# Wait for competing writers instead of failing with SQLITE_BUSY —# multiple proxy workers share this file, and writes are frequentdiff --git a/headroom/copilot_auth.py b/headroom/copilot_auth.pyindex fd9a5504c..ac44b75e9 100644--- a/headroom/copilot_auth.py+++ b/headroom/copilot_auth.py@@ -9,6 +9,7 @@ import jsonimport loggingimport mathimport os+import tempfileimport timefrom collections.abc import Mappingfrom contextvars import ContextVar@@ -623,6 +624,39 @@ def read_headroom_copilot_oauth_token() -> str | None:return token.strip() if isinstance(token, str) and token.strip() else None+def _write_private_text(path: Path, text: str) -> None:+ """Atomically write ``text`` to ``path`` so it is never world/group readable.++ The token file holds a GitHub Copilot OAuth refresh token. Rather than+ create-or-narrow the destination in place — which fails *open* if a pre-write+ ``chmod`` is refused (the secret still lands in a wide file), and is exposed+ to a symlink/path-replacement race between the check and the ``open`` — write+ the secret to a fresh private temp file and atomically rename it into place:++ * ``tempfile.mkstemp`` creates the temp with ``0o600`` and ``O_EXCL`` (it+ never follows a symlink and never reuses an attacker-planted file), so the+ secret is private from birth.+ * ``os.replace`` swaps it into place atomically; the destination inherits the+ temp's ``0o600`` mode. The existing file is never opened, ``chmod``-ed, or+ truncated, so a permission/platform error fails **closed** — it raises+ before the old file is touched (old contents preserved) and the temp is+ cleaned up, rather than leaving a secret in a readable file.++ On Windows POSIX bits do not apply, but ``mkstemp`` still restricts the file+ to the owner and ``os.replace`` is atomic.+ """+ path.parent.mkdir(parents=True, exist_ok=True)+ fd, tmp_name = tempfile.mkstemp(dir=str(path.parent), prefix=f".{path.name}.", suffix=".tmp")+ tmp = Path(tmp_name)+ try:+ with os.fdopen(fd, "w", encoding="utf-8") as handle:+ handle.write(text)+ os.replace(tmp, path)+ except BaseException:+ tmp.unlink(missing_ok=True)+ raise++def save_headroom_copilot_oauth_token(token: str,*,@@ -643,11 +677,7 @@ def save_headroom_copilot_oauth_token("domain": _github_oauth_domain(domain),"created_at": int(time.time()),}- path.write_text(json.dumps(body, indent=2, sort_keys=True) + "\n", encoding="utf-8")- try:- path.chmod(0o600)- except OSError:- pass+ _write_private_text(path, json.dumps(body, indent=2, sort_keys=True) + "\n")return pathdiff --git a/headroom/providers/codex/recovery.py b/headroom/providers/codex/recovery.pyindex 330bbe99d..4d94dec0a 100644--- a/headroom/providers/codex/recovery.py+++ b/headroom/providers/codex/recovery.py@@ -404,6 +404,32 @@ def _quote(identifier: str) -> str:return '"' + identifier.replace('"', '""') + '"'+def _write_private_text(path: Path, text: str) -> None:+ """Atomically write ``text`` to ``path`` without a world/group-readable window.++ The backup manifest records the recovered account's file layout. Rather than+ create-or-narrow the destination in place — which fails *open* if a pre-write+ ``chmod`` is refused, and is exposed to a symlink swap between the check and+ the ``open`` — write to a fresh private temp file and atomically rename it in:+ ``tempfile.mkstemp`` creates it ``0o600`` with ``O_EXCL`` (never follows a+ symlink, never reuses a planted file), and ``os.replace`` swaps it in+ atomically, the destination inheriting ``0o600``. The existing file is never+ opened or truncated, so a permission/platform error fails **closed** — it+ raises before the old file is touched, and the temp is cleaned up. Mirrors the+ Copilot auth writer.+ """+ path.parent.mkdir(parents=True, exist_ok=True)+ fd, tmp_name = tempfile.mkstemp(dir=str(path.parent), prefix=f".{path.name}.", suffix=".tmp")+ tmp = Path(tmp_name)+ try:+ with os.fdopen(fd, "w", encoding="utf-8") as handle:+ handle.write(text)+ os.replace(tmp, path)+ except BaseException:+ tmp.unlink(missing_ok=True)+ raise++def _database_schema(connection: sqlite3.Connection, schema: str) -> dict[str, str]:rows = connection.execute(f"SELECT name, sql FROM {_quote(schema)}.sqlite_master "@@ -741,8 +767,5 @@ def recover_codex_home(*, source: Path, target: Path) -> RecoveryReport:manifest = asdict(report)manifest.update(source=str(source), target=str(target), backup_dir=str(backup_dir))manifest_file = backup_dir / "manifest.json"- manifest_file.write_text(- json.dumps(manifest, indent=2, sort_keys=True) + "\n", encoding="utf-8"- )- manifest_file.chmod(0o600)+ _write_private_text(manifest_file, json.dumps(manifest, indent=2, sort_keys=True) + "\n")return reportdiff --git a/tests/test_ccr_sqlite_backend.py b/tests/test_ccr_sqlite_backend.pyindex 6ca750dea..d7f6384d9 100644--- a/tests/test_ccr_sqlite_backend.py+++ b/tests/test_ccr_sqlite_backend.py@@ -58,6 +58,79 @@ class TestSQLiteBackend:assert not b.exists("h1")assert not b.delete("h1")+ @pytest.mark.skipif(os.name == "nt", reason="POSIX permission bits only")+ def test_db_file_is_private_under_permissive_umask(self, db_path):+ """The CCR db (sensitive tool output) must be 0o600 under any umask.++ ``_ensure_private`` creates the db with an explicit ``0o600`` mode+ (``O_CREAT | O_EXCL``) before ``sqlite3.connect`` opens it, so it is+ private from birth. Forcing ``umask(0o022)`` — which would make sqlite+ create it 0o644 — confirms the mode is umask-independent.+ """+ old_umask = os.umask(0o022)+ try:+ SQLiteBackend(db_path) # fresh open, private from birth+ finally:+ os.umask(old_umask)++ assert db_path.exists()+ assert (db_path.stat().st_mode & 0o777) == 0o600++ @pytest.mark.skipif(os.name == "nt", reason="POSIX chmod semantics")+ def test_open_fails_closed_when_db_cannot_be_made_private(self, db_path, monkeypatch):+ """If an existing db cannot be narrowed, opening must abort (fail closed).++ A store of raw tool output must not be opened world-readable, so+ ``_ensure_private`` raises rather than silently proceeding to+ ``sqlite3.connect`` on a db it could not make private.+ """+ import headroom.cache.backends.sqlite as sqlite_mod++ db_path.write_text("") # pre-existing regular db file++ def boom(fd, mode):+ raise PermissionError("cannot fchmod")++ # The existing-file path narrows through the descriptor (fchmod), so a+ # narrowing failure there must still abort the open.+ monkeypatch.setattr(sqlite_mod.os, "fchmod", boom)+ with pytest.raises(PermissionError):+ SQLiteBackend(db_path)++ @pytest.mark.skipif(os.name == "nt", reason="POSIX symlink semantics")+ def test_open_fails_closed_on_symlinked_db_path(self, tmp_path):+ """A symlinked db path must be refused without following it.++ If the db path is a symlink an attacker planted, ``_ensure_private``+ must fail closed (O_NOFOLLOW) rather than chmod-ing and handing the+ attacker-chosen target to ``sqlite3.connect``. The target must be left+ untouched: neither narrowed nor opened/rewritten.+ """+ target = tmp_path / "attacker_target"+ # Create the target with a deliberately wider mode via umask (not an+ # explicit permissive chmod) so a chmod-follow regression would show up+ # as a mode change, without the test itself performing a world-readable+ # chmod that the security scanner flags as overly permissive.+ old_umask = os.umask(0o022)+ try:+ target.write_text("SECRET-ORIGINAL") # lands 0o644 under this umask+ finally:+ os.umask(old_umask)+ before_mode = target.stat().st_mode & 0o777+ assert before_mode != 0o600 # precondition: a narrow would be visible++ link = tmp_path / "ccr_store.db"+ link.symlink_to(target)++ with pytest.raises(PermissionError):+ SQLiteBackend(link)++ # Target was neither narrowed (mode unchanged) nor opened/rewritten by+ # sqlite (contents intact, no WAL/journal siblings created).+ assert (target.stat().st_mode & 0o777) == before_mode+ assert target.read_text() == "SECRET-ORIGINAL"+ assert link.is_symlink()+def test_survives_reopen(self, db_path):"""The restart-survival property the default flip exists for."""SQLiteBackend(db_path).set("h1", make_entry())diff --git a/tests/test_cli/test_recover_codex.py b/tests/test_cli/test_recover_codex.pyindex dea46a503..c8d50e8f0 100644--- a/tests/test_cli/test_recover_codex.py+++ b/tests/test_cli/test_recover_codex.py@@ -18,6 +18,50 @@ from headroom.cli.main import mainfrom headroom.providers.codex.recovery import discover_dangling_homes, recover_codex_home[email protected](os.name == "nt", reason="POSIX permission bits only")+def test_write_private_text_is_private_under_permissive_umask(tmp_path: Path) -> None:+ """The backup manifest must be 0o600 regardless of umask.++ The atomic writer creates the file via ``tempfile.mkstemp`` (always 0o600)+ and ``os.replace``s it into place, so the mode does not depend on umask.+ Forcing ``umask(0o022)`` (which would make a plain ``write_text`` land 0o644)+ confirms the file is private from birth.+ """+ target = tmp_path / "manifest.json"+ old_umask = os.umask(0o022)+ try:+ codex_recovery._write_private_text(target, '{"ok": true}\n')+ finally:+ os.umask(old_umask)++ assert target.read_text(encoding="utf-8") == '{"ok": true}\n'+ assert (target.stat().st_mode & 0o777) == 0o600+++def test_write_private_text_fails_closed_preserving_existing(+ tmp_path: Path, monkeypatch: pytest.MonkeyPatch+) -> None:+ """A failed rename must not truncate or replace an existing manifest.++ The write lands in a private temp file and is atomically renamed in, so a+ failure at ``os.replace`` leaves the old contents intact and cleans up the+ temp — never a partial or world-readable secret in place.+ """+ target = tmp_path / "manifest.json"+ codex_recovery._write_private_text(target, '{"v": 1}\n')+ before = set(os.listdir(tmp_path))++ def boom(*_args: object, **_kwargs: object) -> None:+ raise OSError("simulated rename failure")++ monkeypatch.setattr(codex_recovery.os, "replace", boom)+ with pytest.raises(OSError):+ codex_recovery._write_private_text(target, '{"v": 2}\n')++ assert target.read_text(encoding="utf-8") == '{"v": 1}\n'+ assert set(os.listdir(tmp_path)) == before++def _write_db(path: Path, rows: list[tuple[str, str]]) -> None:path.parent.mkdir(parents=True, exist_ok=True)connection = sqlite3.connect(path)diff --git a/tests/test_copilot_auth.py b/tests/test_copilot_auth.pyindex 70190b862..00581698d 100644--- a/tests/test_copilot_auth.py+++ b/tests/test_copilot_auth.py@@ -2,6 +2,7 @@ from __future__ import annotationsimport asyncioimport json+import osimport timefrom pathlib import Pathfrom types import SimpleNamespace@@ -93,6 +94,63 @@ def test_read_cached_oauth_token_prefers_headroom_login(assert copilot_auth.read_cached_oauth_token() == "gho-headroom"+def test_saved_oauth_token_file_content_roundtrips(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ """The token still writes and reads back after the private-write change."""+ copilot_auth.save_headroom_copilot_oauth_token("gho-headroom")+ assert copilot_auth.read_cached_oauth_token() == "gho-headroom"+ # Re-saving over an existing file rotates the token cleanly.+ copilot_auth.save_headroom_copilot_oauth_token("gho-rotated")+ assert copilot_auth.read_cached_oauth_token() == "gho-rotated"++[email protected](os.name == "nt", reason="POSIX permission bits only")+def test_saved_oauth_token_file_is_private_under_permissive_umask() -> None:+ """The token file must be 0o600 even under a permissive umask.++ The atomic writer creates the file via ``tempfile.mkstemp`` (always 0o600,+ umask-independent) and ``os.replace``s it into place, so the destination is+ private from birth with no world-readable window. Forcing ``umask(0o022)``+ (which would make a plain ``write_text`` land 0o644) confirms the mode does+ not depend on umask.+ """+ old_umask = os.umask(0o022)+ try:+ path = Path(copilot_auth.save_headroom_copilot_oauth_token("gho-headroom"))+ finally:+ os.umask(old_umask)++ assert path.exists()+ assert (path.stat().st_mode & 0o777) == 0o600+++def test_token_write_failure_preserves_existing_token(monkeypatch: pytest.MonkeyPatch) -> None:+ """A failed write must fail CLOSED: the old token survives, no secret leaks.++ The write goes to a private temp file and is atomically renamed in, so a+ failure at the rename never truncates or replaces the existing file (unlike+ the earlier create-or-narrow-in-place approach, which opened the destination+ ``O_TRUNC`` before writing). The temp file is cleaned up so no secret is left+ behind in a stray path.+ """+ copilot_auth.save_headroom_copilot_oauth_token("gho-original")+ path = copilot_auth.headroom_copilot_auth_path()+ before = set(os.listdir(path.parent))++ monkeypatch.setattr(copilot_auth.os, "replace", _boom)+ with pytest.raises(OSError):+ copilot_auth.save_headroom_copilot_oauth_token("gho-should-not-land")++ # Old token intact; no partial/temp file left behind.+ assert copilot_auth.read_cached_oauth_token() == "gho-original"+ assert set(os.listdir(path.parent)) == before+++def _boom(*_args: object, **_kwargs: object) -> None:+ raise OSError("simulated write failure")++def test_default_oauth_domain_uses_enterprise_url_host(monkeypatch: pytest.MonkeyPatch) -> None:monkeypatch.setenv("GITHUB_COPILOT_ENTERPRISE_URL", "https://ghe.example.com")