Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

fix(security): create memory stores and other state files owner-only (#3848)

Tejas Chopra5119b6eb545e
diff --git a/headroom/cache/backends/sqlite.py b/headroom/cache/backends/sqlite.py
index 6d0ca410d..13d5386f1 100644
--- a/headroom/cache/backends/sqlite.py
+++ b/headroom/cache/backends/sqlite.py
@@ -22,13 +22,14 @@ import json
import logging
import os
import sqlite3
-import stat
import threading
import time
from dataclasses import asdict, fields
from pathlib import Path
from typing import TYPE_CHECKING, Any
+from ...fileperms import ensure_private_file
+
if TYPE_CHECKING:
from ..compression_store import CompressionEntry
@@ -90,65 +91,13 @@ class SQLiteBackend:
The originals stored here can contain sensitive tool output (file
contents, command output). ``mkdir`` created the parent at the umask
- default (typically world-traversable ``0o755``), and sqlite would
- otherwise create the db file itself at the umask default too — leaving it
- world/group readable. This creates the file with an explicit ``0o600``
- mode before ``sqlite3.connect`` runs (a *new* db is private from birth;
- sqlite treats the empty file as a fresh database), or narrows a
- pre-existing db.
-
- A store of raw tool output must not be opened world-readable, so a
- failure to create or narrow it privately **raises** rather than silently
- proceeding to open a wide file.
-
- The existing-file path is symlink-race resistant. ``O_EXCL`` on the
- create refuses to reuse an attacker-planted file or symlink. When the
- file already exists it is re-opened with ``O_NOFOLLOW`` and narrowed
- through that descriptor (``fstat`` to confirm a regular file, ``fchmod``
- to set the mode) rather than by re-resolving the path: a symlink is
- refused at the ``open`` syscall, and operating on the verified
- descriptor closes the check-then-chmod TOCTOU that a ``chmod(path)``
- (which follows symlinks) would leave open. On Windows POSIX permission
- bits and ``O_NOFOLLOW`` do not apply, so there is nothing to narrow.
+ default and sqlite would create the db at the umask default too, so the
+ file is created (or an existing one narrowed) through
+ :func:`headroom.fileperms.ensure_private_file`, which is symlink-race
+ resistant and raises :class:`PermissionError` rather than opening a
+ wide file. See that function for the exact guarantee per platform.
"""
- # Fast path: create the file ourselves. O_EXCL (plus O_NOFOLLOW where
- # available) refuses to reuse an existing file or follow a planted
- # symlink, so a brand-new db is private from birth at 0o600.
- create_flags = os.O_CREAT | os.O_EXCL | os.O_WRONLY
- if hasattr(os, "O_NOFOLLOW"):
- create_flags |= os.O_NOFOLLOW
- try:
- os.close(os.open(path, create_flags, 0o600))
- return
- except FileExistsError:
- pass
-
- # The file already exists. Without POSIX no-follow/fd primitives
- # (Windows) permission bits do not apply and there is no symlink-race
- # hardening to perform; sqlite opens by path as before.
- if not (hasattr(os, "O_NOFOLLOW") and hasattr(os, "fchmod")):
- return
-
- # Narrow the existing file, but only after proving — through a single
- # no-follow descriptor — that it is a real regular file. A symlink is
- # refused by O_NOFOLLOW (fail closed), and chmod-ing the descriptor
- # rather than the path means an attacker cannot swap the inode we
- # verified between the check and the narrow.
- try:
- fd = os.open(path, os.O_RDONLY | os.O_NOFOLLOW)
- except OSError as exc:
- raise PermissionError(
- f"refusing to open CCR store at {path}: not a regular file "
- f"(symlink or open error: {exc})"
- ) from exc
- try:
- if not stat.S_ISREG(os.fstat(fd).st_mode):
- raise PermissionError(f"refusing to open CCR store at {path}: not a regular file")
- # os.fchmod is POSIX-only (guarded by the hasattr check above); the
- # ignore keeps type-checking clean on Windows where it is absent.
- os.fchmod(fd, 0o600) # type: ignore[attr-defined]
- finally:
- os.close(fd)
+ ensure_private_file(path, what="CCR store")
def _open(self) -> sqlite3.Connection:
self._ensure_private(self._path)
diff --git a/headroom/fileperms.py b/headroom/fileperms.py
index bb4a9b137..f5e76daab 100644
--- a/headroom/fileperms.py
+++ b/headroom/fileperms.py
@@ -3,8 +3,22 @@
Headroom's runtime log (``~/.headroom/logs/proxy-<port>.log``) and the optional
JSONL request log can both carry verbatim request and response content: wire
debug dumps, ``--log-messages`` bodies, and — when an operator opts in — CCR
-payload previews. None of that should be created at the process umask, which on
-a stock developer machine means ``0644``: world-readable.
+payload previews. The CCR retrieval store holds verbatim tool output, and the
+memory databases hold facts extracted from conversations together with the
+user id they belong to. None of that should be created at the process umask,
+which on a stock developer machine means ``0644``: world-readable.
+
+This module is the one place that knows how to create such a file privately,
+so every store uses the same rules and a reviewer can read the guarantee off
+one file. Extensions that keep their own state (Shield, Foresight, Kiro, the
+router) should call these helpers rather than ``sqlite3.connect`` /
+``Path.write_text`` directly:
+
+* :func:`open_owner_only` — open a log or text file for append or write.
+* :func:`ensure_private_file` — make a path a private regular file *before*
+ a library that opens by path (sqlite) touches it.
+* :func:`connect_private_sqlite` — ``sqlite3.connect`` through the above.
+* :func:`private_dir` — create a directory that will hold such files.
Scope of the guarantee — read this before citing it in a threat model:
@@ -33,20 +47,27 @@ a kernel-enforced one.
from __future__ import annotations
+import functools
import os
+import sqlite3
import stat
+from pathlib import Path
from typing import IO, Any
+from urllib.parse import parse_qs, unquote, urlsplit
#: Mode for every runtime file Headroom creates that may hold request content.
OWNER_ONLY_MODE = 0o600
+#: Mode for directories that hold such files.
+OWNER_ONLY_DIR_MODE = 0o700
+
#: ``True`` only where the mode bits above actually decide who can read the
#: file. See the module docstring for why Windows is excluded.
OWNER_ONLY_SUPPORTED = os.name == "posix"
-def _open_flags() -> int:
- flags = os.O_CREAT | os.O_WRONLY | os.O_APPEND
+def _open_flags(*, truncate: bool = False) -> int:
+ flags = os.O_CREAT | os.O_WRONLY | (os.O_TRUNC if truncate else os.O_APPEND)
# Refuse to open through a symlink where the platform can enforce it, so a
# planted link cannot redirect either the write or the chmod. Absent on
# Windows, where getattr() leaves the flag out.
@@ -97,18 +118,21 @@ def open_owner_only(
*,
encoding: str | None = None,
errors: str | None = None,
+ newline: str | None = None,
) -> IO[Any]:
- """Open *path* for append, creating it owner-only.
+ """Open *path* for append (``"a"``) or write (``"w"``), creating it owner-only.
Raises ``OSError`` — which every caller already treats as "logging is
unavailable, carry on" — if the path cannot be opened, including when it is
a symlink on a platform with ``O_NOFOLLOW``. Failing closed is deliberate:
a redirected sensitive log is worse than no log.
"""
- fd = os.open(path, _open_flags(), OWNER_ONLY_MODE)
+ if mode[:1] not in ("a", "w"):
+ raise ValueError(f"open_owner_only: mode must start with 'a' or 'w', got {mode!r}")
+ fd = os.open(path, _open_flags(truncate=mode.startswith("w")), OWNER_ONLY_MODE)
try:
restrict_fd_to_owner(fd)
- return open(fd, mode, encoding=encoding, errors=errors, closefd=True)
+ return open(fd, mode, encoding=encoding, errors=errors, newline=newline, closefd=True)
except BaseException:
try:
os.close(fd)
@@ -116,3 +140,157 @@ def open_owner_only(
# open() can have taken and closed the descriptor on its way out.
pass
raise
+
+
+def ensure_private_file(path: str | os.PathLike[str], *, what: str = "file") -> None:
+ """Make *path* an owner-only regular file before a by-path opener touches it.
+
+ For libraries that open files by path and create them at the umask —
+ sqlite above all. Creating the file here first, with an explicit
+ ``0o600`` mode, means a *new* database is private from birth (sqlite
+ treats an empty file as a fresh database, and creates its ``-wal`` /
+ ``-shm`` sidecars with the same mode as the main file). A pre-existing
+ file left wide by an earlier run is narrowed.
+
+ Fails **closed**: a store of conversation or tool content must not be
+ opened world-readable, so a failure to create or narrow the file privately
+ raises :class:`PermissionError` rather than proceeding to open a wide
+ file. The existing-file path is symlink-race resistant — ``O_EXCL`` on the
+ create refuses to reuse a planted file or symlink; an existing file is
+ re-opened with ``O_NOFOLLOW`` and narrowed through that descriptor
+ (``fstat`` to confirm a regular file, ``fchmod`` to set the mode) rather
+ than by re-resolving the path, which closes the check-then-chmod TOCTOU a
+ ``chmod(path)`` would leave open. On Windows POSIX mode bits and
+ ``O_NOFOLLOW`` do not apply, so there is nothing to narrow (see the module
+ docstring for what is and is not claimed there).
+
+ *what* names the store in the error message.
+ """
+ create_flags = os.O_CREAT | os.O_EXCL | os.O_WRONLY
+ if hasattr(os, "O_NOFOLLOW"):
+ create_flags |= os.O_NOFOLLOW
+ try:
+ os.close(os.open(path, create_flags, OWNER_ONLY_MODE))
+ return
+ except FileExistsError:
+ pass
+
+ if not (hasattr(os, "O_NOFOLLOW") and hasattr(os, "fchmod")):
+ return
+
+ try:
+ fd = os.open(path, os.O_RDONLY | os.O_NOFOLLOW)
+ except OSError as exc:
+ raise PermissionError(
+ f"refusing to open {what} at {path}: not a regular file (symlink or open error: {exc})"
+ ) from exc
+ try:
+ if not stat.S_ISREG(os.fstat(fd).st_mode):
+ raise PermissionError(f"refusing to open {what} at {path}: not a regular file")
+ # os.fchmod is POSIX-only (guarded by the hasattr check above); the
+ # ignore keeps type-checking clean on Windows where it is absent.
+ os.fchmod(fd, OWNER_ONLY_MODE) # type: ignore[attr-defined]
+ finally:
+ os.close(fd)
+
+
+def _sqlite_always_uri() -> bool:
+ """Whether this sqlite reads ``file:`` names as URIs even without ``uri=True``.
+
+ True for builds compiled with ``SQLITE_USE_URI`` — Debian's, and so the
+ official ``python`` Docker images.
+ """
+ conn = sqlite3.connect(":memory:")
+ try:
+ options = {row[0] for row in conn.execute("PRAGMA compile_options")}
+ finally:
+ conn.close()
+ return bool(options & {"USE_URI", "USE_URI=1"})
+
+
+def _sqlite_file_path(path: str | os.PathLike[str], *, uri: bool) -> str | None:
+ """The file sqlite will create or open for *path*; ``None`` if there is none.
+
+ Mirrors sqlite's own rules. Without ``uri=True`` the name is a literal
+ path, even one that starts with ``file:``, unless this sqlite was built to
+ always parse URIs (:func:`_sqlite_always_uri`). Parsed as a URI, ``file:`` names
+ its percent-decoded path (relative to the working directory unless
+ absolute), and is in-memory when that path is empty or ``:memory:`` or the
+ query says ``mode=memory``. ``mode=ro`` / ``mode=rw`` never create the
+ file, so a missing one is left for sqlite to report.
+ """
+ text = os.fspath(path)
+ if text in ("", ":memory:"):
+ return None
+ if not (text.startswith("file:") and (uri or _sqlite_always_uri())):
+ return text
+ if not OWNER_ONLY_SUPPORTED:
+ # No mode bits to set (module docstring), and Windows URIs carry a
+ # drive letter sqlite strips its own way: pass them through.
+ return None
+ parts = urlsplit(text)
+ # surrogateescape keeps an undecodable %XX byte as that raw byte, which is
+ # the name sqlite opens; the default "replace" would secure a different one.
+ file_path = unquote(parts.path, errors="surrogateescape")
+ mode = parse_qs(parts.query).get("mode", [""])[-1]
+ if file_path in ("", ":memory:") or mode == "memory":
+ return None
+ if parts.netloc not in ("", "localhost"):
+ raise PermissionError(f"refusing SQLite URI with a remote authority: {text}")
+ if mode in ("ro", "rw") and not os.path.lexists(file_path):
+ return None
+ return file_path
+
+
+#: Files sqlite keeps next to a database and reopens by name.
+_SQLITE_SIDECAR_SUFFIXES = ("-wal", "-shm", "-journal")
+
+
+def connect_private_sqlite(
+ path: str | os.PathLike[str],
+ *,
+ what: str = "database",
+ **connect_kwargs: Any,
+) -> sqlite3.Connection:
+ """``sqlite3.connect`` that first makes the database file owner-only.
+
+ Drop-in for ``sqlite3.connect(str(path), **kwargs)`` at every store that
+ holds conversation-derived content. In-memory databases are passed
+ straight through; a file-backed ``file:`` URI (``uri=True``) has its
+ underlying file made private like a plain path. Existing ``-wal`` /
+ ``-shm`` / ``-journal`` sidecars are narrowed too. The parent directory must
+ already exist; create it with :func:`private_dir` if it is dedicated to
+ this store.
+ """
+ file_path = _sqlite_file_path(path, uri=bool(connect_kwargs.get("uri")))
+ if file_path is not None:
+ ensure_private_file(file_path, what=what)
+ # A crash can leave -wal/-shm/-journal sidecars behind with the mode
+ # of an older, unhardened run, and sqlite reuses them as they are.
+ # Narrow any that exist (refusing symlinks) before sqlite writes to
+ # them. Missing ones are left alone: sqlite creates them with the
+ # main file's mode, which is now 0600.
+ for suffix in _SQLITE_SIDECAR_SUFFIXES:
+ sidecar = file_path + suffix
+ if os.path.lexists(sidecar):
+ ensure_private_file(sidecar, what=f"{what} {suffix[1:]} file")
+ # ``**connect_kwargs: Any`` makes mypy type the call as ``Any``; the
+ # annotation pins it back to the real return type.
+ conn: sqlite3.Connection = sqlite3.connect(os.fspath(path), **connect_kwargs)
+ return conn
+
+
+def private_dir(path: str | os.PathLike[str]) -> Path:
+ """Create *path* (and missing parents) and make the leaf directory ``0o700``.
+
+ For a directory that exists only to hold Headroom's sensitive files (the
+ native memory directory); an existing directory left wider by an earlier
+ run is narrowed. A symlink at the leaf is left alone: the target is not
+ ours to re-permission. Returns the path.
+ """
+ target = Path(path)
+ target.mkdir(parents=True, exist_ok=True)
+ if OWNER_ONLY_SUPPORTED and not target.is_symlink():
+ os.chmod(target, OWNER_ONLY_DIR_MODE)
+ return target
diff --git a/headroom/memory/adapters/fts5.py b/headroom/memory/adapters/fts5.py
index a0920a444..a93d8b3c9 100644
--- a/headroom/memory/adapters/fts5.py
+++ b/headroom/memory/adapters/fts5.py
@@ -14,6 +14,7 @@ from dataclasses import dataclass, field
from pathlib import Path
from typing import TYPE_CHECKING, Any
+from ...fileperms import connect_private_sqlite
from ..models import Memory
from ..ports import TextFilter, TextSearchResult
@@ -75,7 +76,7 @@ class FTS5TextIndex:
Commits on clean exit, rolls back on exception, and always closes
the connection -- callers use ``with self._get_conn() as conn:``.
"""
- conn = sqlite3.connect(str(self.db_path))
+ conn = connect_private_sqlite(self.db_path, what="memory text index")
conn.row_factory = sqlite3.Row
try:
with conn:
diff --git a/headroom/memory/adapters/hnsw.py b/headroom/memory/adapters/hnsw.py
index 48e087c64..a8c03d2f4 100644
--- a/headroom/memory/adapters/hnsw.py
+++ b/headroom/memory/adapters/hnsw.py
@@ -23,6 +23,7 @@ from typing import TYPE_CHECKING, Any
import numpy as np
+from ...fileperms import ensure_private_file, open_owner_only
from ..models import Memory, ScopeLevel, normalize_entity_refs
from ..ports import VectorFilter, VectorSearchResult
@@ -812,8 +813,11 @@ class HNSWVectorIndex:
path = Path(path)
with self._lock:
- # Save HNSW index
+ # Save HNSW index. hnswlib opens the path itself and would create
+ # it at the umask, so make it a private regular file first (refusing
+ # a symlink); its truncating write keeps the 0600 mode.
hnsw_path = path.with_suffix(".hnsw")
+ ensure_private_file(hnsw_path, what="HNSW index")
self._index.save_index(str(hnsw_path))
# Save metadata, mappings, and embeddings
@@ -834,7 +838,9 @@ class HNSWVectorIndex:
"embeddings": {mid: emb.tolist() for mid, emb in self._embeddings.items()},
}
- with open(meta_path, "w") as f:
+ # Memory metadata and raw embeddings: owner-only, like every other
+ # memory store file.
+ with open_owner_only(meta_path, "w") as f:
json.dump(meta_data, f)
def load_index(self, path: str | Path) -> None:
diff --git a/headroom/memory/adapters/sqlite.py b/headroom/memory/adapters/sqlite.py
index 5d3289f24..4a5b339ae 100644
--- a/headroom/memory/adapters/sqlite.py
+++ b/headroom/memory/adapters/sqlite.py
@@ -17,6 +17,7 @@ from datetime import datetime, timezone
from pathlib import Path
from typing import TYPE_CHECKING, Any
+from ...fileperms import connect_private_sqlite
from ..models import Memory, ScopeLevel, normalize_entity_refs
from ..ports import MemoryFilter
@@ -82,8 +83,10 @@ class SQLiteMemoryStore:
Commits on clean exit, rolls back on exception, and always closes
the connection -- callers use ``with self._get_conn() as conn:``.
+ The file is created, or narrowed, owner-only before sqlite opens it:
+ it holds memory content and the user ids it belongs to.
"""
- conn = sqlite3.connect(str(self.db_path))
+ conn = connect_private_sqlite(self.db_path, what="memory store")
conn.row_factory = sqlite3.Row
try:
with conn:
diff --git a/headroom/memory/adapters/sqlite_graph.py b/headroom/memory/adapters/sqlite_graph.py
index 719855c3b..710440fd8 100644
--- a/headroom/memory/adapters/sqlite_graph.py
+++ b/headroom/memory/adapters/sqlite_graph.py
@@ -23,6 +23,7 @@ from pathlib import Path
from threading import RLock
from typing import TYPE_CHECKING, Any
+from ...fileperms import connect_private_sqlite
from .graph_models import Entity, Relationship, RelationshipDirection, Subgraph
if TYPE_CHECKING:
@@ -84,7 +85,7 @@ class SQLiteGraphStore:
Commits on clean exit, rolls back on exception, and always closes
the connection -- callers use ``with self._get_conn() as conn:``.
"""
- conn = sqlite3.connect(str(self.db_path))
+ conn = connect_private_sqlite(self.db_path, what="memory graph store")
conn.row_factory = sqlite3.Row
# Configure page cache size (negative = KB, positive = pages)
diff --git a/headroom/memory/adapters/sqlite_vector.py b/headroom/memory/adapters/sqlite_vector.py
index 628ca174e..2db269923 100644
--- a/headroom/memory/adapters/sqlite_vector.py
+++ b/headroom/memory/adapters/sqlite_vector.py
@@ -29,6 +29,7 @@ from typing import TYPE_CHECKING, Any, cast
import numpy as np
+from ...fileperms import connect_private_sqlite
from ..models import Memory, ScopeLevel, normalize_entity_refs
from ..ports import VectorFilter, VectorSearchResult
@@ -235,8 +236,8 @@ class SQLiteVectorIndex:
self._init_db()
def _create_conn(self) -> sqlite3.Connection:
- """Create a SQLite connection with sqlite-vec loaded."""
- conn = sqlite3.connect(str(self._db_path))
+ """Create a SQLite connection with sqlite-vec loaded (file owner-only)."""
+ conn = connect_private_sqlite(self._db_path, what="memory vector index")
conn.row_factory = sqlite3.Row
# Load sqlite-vec extension
diff --git a/headroom/memory/traffic_learner.py b/headroom/memory/traffic_learner.py
index 5818abce7..468071369 100644
--- a/headroom/memory/traffic_learner.py
+++ b/headroom/memory/traffic_learner.py
@@ -1511,7 +1511,9 @@ class TrafficLearner:
now_iso = datetime.now(timezone.utc).isoformat()
def _bump() -> bool:
- conn = sqlite3.connect(str(db_path))
+ from ..fileperms import connect_private_sqlite
+
+ conn = connect_private_sqlite(db_path, what="memory store")
try:
cursor = conn.execute(
"UPDATE memories SET metadata = json_set("
diff --git a/headroom/proxy/memory_handler.py b/headroom/proxy/memory_handler.py
index 5f6c0c891..9b245c51f 100644
--- a/headroom/proxy/memory_handler.py
+++ b/headroom/proxy/memory_handler.py
@@ -241,8 +241,11 @@ class MemoryHandler:
self._native_memory_dir = _paths.native_memory_dir()
- # Create directory if it doesn't exist
- self._native_memory_dir.mkdir(parents=True, exist_ok=True)
+ # Create the directory owner-only (and narrow it if an earlier run left
+ # it wider): it holds the model's memory files verbatim.
+ from headroom import fileperms as _fileperms
+
+ _fileperms.private_dir(self._native_memory_dir)
logger.info(f"Memory: Native memory directory: {self._native_memory_dir}")
def get_beta_headers(self) -> dict[str, str]:
diff --git a/headroom/telemetry/reporter.py b/headroom/telemetry/reporter.py
index b7f39118c..22b31214c 100644
--- a/headroom/telemetry/reporter.py
+++ b/headroom/telemetry/reporter.py
@@ -29,6 +29,7 @@ from typing import TYPE_CHECKING, Any
import httpx
+from headroom import fileperms as _fileperms
from headroom import paths as _paths
if TYPE_CHECKING:
@@ -365,9 +366,12 @@ class UsageReporter:
return
try:
self._cache_path.parent.mkdir(parents=True, exist_ok=True)
- self._cache_path.write_text(
- json.dumps(self._license_info.to_dict(), indent=2), encoding="utf-8"
- )
+ # Owner-only: the envelope names the org and plan.
+ # newline pinned: state files are LF on every platform (#3698).
+ with _fileperms.open_owner_only(
+ self._cache_path, "w", encoding="utf-8", newline="\n"
+ ) as fh:
+ fh.write(json.dumps(self._license_info.to_dict(), indent=2))
except OSError:
logger.warning("Could not save license cache to %s", self._cache_path)
diff --git a/tests/test_private_state_files.py b/tests/test_private_state_files.py
new file mode 100644
index 000000000..995675ad9
--- /dev/null
+++ b/tests/test_private_state_files.py
@@ -0,0 +1,413 @@
+"""Every state file that holds conversation-derived content is created owner-only.
+
+The CCR store and the runtime log already were; the memory databases (facts
+extracted from conversations plus the user id they belong to), the vector and
+graph stores, the HNSW metadata dump, the licence validation cache and the
+native memory directory were created at the process umask — ``0644`` on a
+stock host, readable by any local account.
+
+The fix routes them all through ``headroom.fileperms``, so these tests pin
+both the primitives and every call site, under a permissive umask so the
+result is known to be umask-independent.
+"""
+
+from __future__ import annotations
+
+import os
+import sqlite3
+import stat
+import subprocess
+import sys
+
+import pytest
+
+from headroom import fileperms
+
+pytestmark = pytest.mark.skipif(
+ not fileperms.OWNER_ONLY_SUPPORTED, reason="POSIX permission bits only"
+)
+
+
+def _mode(path) -> int:
+ return stat.S_IMODE(os.stat(path).st_mode)
+
+
[email protected](autouse=True)
+def _permissive_umask():
+ old = os.umask(0o022)
+ try:
+ yield
+ finally:
+ os.umask(old)
+
+
+# ---- primitives -------------------------------------------------------------
+
+
+class TestEnsurePrivateFile:
+ def test_new_file_is_private_from_birth(self, tmp_path):
+ p = tmp_path / "store.db"
+ fileperms.ensure_private_file(p)
+ assert _mode(p) == 0o600
+
+ def test_existing_wide_file_is_narrowed(self, tmp_path):
+ p = tmp_path / "store.db"
+ p.write_text("x")
+ assert _mode(p) == 0o644 # precondition under the 022 umask
+ fileperms.ensure_private_file(p)
+ assert _mode(p) == 0o600
+ assert p.read_text() == "x"
+
+ def test_symlink_is_refused_and_target_untouched(self, tmp_path):
+ target = tmp_path / "victim"
+ target.write_text("SECRET")
+ before = _mode(target)
+ link = tmp_path / "store.db"
+ link.symlink_to(target)
+ with pytest.raises(PermissionError):
+ fileperms.ensure_private_file(link, what="memory store")
+ assert _mode(target) == before
+ assert target.read_text() == "SECRET"
+
+ def test_directory_is_refused(self, tmp_path):
+ d = tmp_path / "store.db"
+ d.mkdir()
+ with pytest.raises(PermissionError):
+ fileperms.ensure_private_file(d)
+
+ def test_narrowing_failure_fails_closed(self, tmp_path, monkeypatch):
+ p = tmp_path / "store.db"
+ p.write_text("")
+
+ def boom(fd, mode):
+ raise PermissionError("cannot fchmod")
+
+ monkeypatch.setattr(os, "fchmod", boom)
+ with pytest.raises(PermissionError):
+ fileperms.ensure_private_file(p)
+
+
+class TestConnectPrivateSqlite:
+ def test_creates_private_db_and_sidecars(self, tmp_path):
+ p = tmp_path / "m.db"
+ conn = fileperms.connect_private_sqlite(p)
+ conn.execute("PRAGMA journal_mode=WAL")
+ conn.execute("CREATE TABLE t(x)")
+ conn.execute("INSERT INTO t VALUES (1)")
+ conn.commit()
+ assert _mode(p) == 0o600
+ # sqlite gives the -wal/-shm sidecars the main file's mode.
+ for sidecar in ("m.db-wal", "m.db-shm"):
+ if (tmp_path / sidecar).exists():
+ assert _mode(tmp_path / sidecar) == 0o600
+ conn.close()
+
+ @pytest.mark.parametrize(
+ ("name", "uri"),
+ [
+ (":memory:", False),
+ ("", False),
+ (":memory:", True),
+ ("file::memory:?cache=shared", True),
+ ("file:mem?mode=memory&cache=shared", True),
+ ],
+ )
+ def test_in_memory_databases_create_no_files(self, tmp_path, monkeypatch, name, uri):
+ monkeypatch.chdir(tmp_path)
+ fileperms.connect_private_sqlite(name, uri=uri).close()
+ # No file literally named ":memory:" (a bug another store has shipped).
+ assert sorted(p.name for p in tmp_path.iterdir()) == []
+
+ @pytest.mark.parametrize(
+ ("uri_for", "created"),
+ [
+ # Relative, percent-encoded, with an explicit create mode.
+ (lambda d: "file:my%20memory.db?mode=rwc", "my memory.db"),
+ (lambda d: f"file:{d}/abs.db", "abs.db"),
+ (lambda d: f"file://localhost{d}/host.db?cache=shared", "host.db"),
+ ],
+ ids=["relative-percent-encoded", "absolute", "localhost-authority"],
+ )
+ def test_file_backed_uri_is_private(self, tmp_path, monkeypatch, uri_for, created):
+ monkeypatch.chdir(tmp_path)
+ conn = fileperms.connect_private_sqlite(uri_for(tmp_path), uri=True)
+ conn.execute("CREATE TABLE t(x)")
+ conn.commit()
+ conn.close()
+ # Under the forced 022 umask sqlite alone would have made this 0644.
+ assert _mode(tmp_path / created) == 0o600
+
+ def test_file_prefixed_literal_path_is_private(self, tmp_path, monkeypatch):
+ # Without uri=True sqlite treats "file:..." as an ordinary file name,
+ # unless it was built with SQLITE_USE_URI (Debian, python Docker images).
+ monkeypatch.chdir(tmp_path)
+ conn = fileperms.connect_private_sqlite("file:literal.db")
+ conn.execute("CREATE TABLE t(x)")
+ conn.commit()
+ conn.close()
+ assert [p.name for p in tmp_path.iterdir()] == [
+ "literal.db" if fileperms._sqlite_always_uri() else "file:literal.db"
+ ]
+ assert _mode(tmp_path / os.listdir(tmp_path)[0]) == 0o600
+
+ def test_file_prefix_is_a_uri_on_builds_that_always_parse_uris(self, monkeypatch):
+ monkeypatch.setattr(fileperms, "_sqlite_always_uri", lambda: True)
+ assert fileperms._sqlite_file_path("file:literal.db", uri=False) == "literal.db"
+ assert fileperms._sqlite_file_path("file::memory:", uri=False) is None
+ assert fileperms._sqlite_file_path("plain.db", uri=False) == "plain.db"
+
+ def test_undecodable_uri_escape_secures_the_raw_byte_name(self, tmp_path, monkeypatch):
+ # sqlite opens the raw 0xFF byte; securing U+FFFD instead would leave
+ # the real database at the umask.
+ monkeypatch.chdir(tmp_path)
+ try:
+ conn = fileperms.connect_private_sqlite("file:secret%FF.db", uri=True)
+ except OSError as exc: # e.g. APFS refuses non-UTF-8 names
+ pytest.skip(f"filesystem rejects non-UTF-8 names: {exc}")
+ conn.execute("CREATE TABLE t(x)")
+ conn.commit()
+ conn.close()
+ assert os.listdir(os.fsencode(tmp_path)) == [b"secret\xff.db"]
+ assert _mode(os.path.join(os.fsencode(tmp_path), b"secret\xff.db")) == 0o600
+
+ @pytest.mark.parametrize(
+ ("journal_mode", "sidecars"),
+ [("wal", ("-wal", "-shm")), ("persist", ("-journal",))],
+ )
+ def test_crash_leftover_sidecars_are_narrowed(self, tmp_path, journal_mode, sidecars):
+ # An older, unhardened run writes under the 022 umask and dies without
+ # a checkpoint, leaving 0644 sidecars that sqlite would reuse as-is.
+ db = tmp_path / "old.db"
+ script = (
+ "import os, sqlite3, sys\n"
+ "c = sqlite3.connect(sys.argv[1])\n"
+ f"c.execute('PRAGMA journal_mode={journal_mode}')\n"
+ "c.execute('CREATE TABLE t(x)')\n"
+ "c.execute(\"INSERT INTO t VALUES ('old')\")\n"
+ "c.commit()\n"
+ "os._exit(0)\n"
+ )
+ subprocess.run([sys.executable, "-c", script, str(db)], check=True)
+ for suffix in sidecars:
+ assert _mode(f"{db}{suffix}") == 0o644 # precondition
+
+ conn = fileperms.connect_private_sqlite(db)
+ for suffix in sidecars:
+ assert _mode(f"{db}{suffix}") == 0o600 # before sqlite writes a byte
+ conn.execute(f"PRAGMA journal_mode={journal_mode}")
+ conn.execute("INSERT INTO t VALUES ('PRIVATE-SENTINEL')")
+ conn.commit()
+
+ for suffix in sidecars:
+ assert _mode(f"{db}{suffix}") == 0o600
+ if journal_mode == "wal":
+ # The new row went into the reused WAL, which is now private.
+ assert b"PRIVATE-SENTINEL" in (tmp_path / "old.db-wal").read_bytes()
+ # Narrowing kept the leftover contents: the uncheckpointed row survives.
+ assert conn.execute("SELECT x FROM t ORDER BY rowid").fetchall() == [
+ ("old",),
+ ("PRIVATE-SENTINEL",),
+ ]
+ conn.close()
+
+ def test_missing_sidecars_are_not_precreated(self, tmp_path):
+ db = tmp_path / "m.db"
+ fileperms.connect_private_sqlite(db).close()
+ assert [p.name for p in tmp_path.iterdir()] == ["m.db"]
+
+ @pytest.mark.parametrize("suffix", ["-wal", "-shm", "-journal"])
+ def test_refuses_symlinked_sidecar(self, tmp_path, suffix):
+ db = tmp_path / "m.db"
+ sqlite3.connect(db).close()
+ victim = tmp_path / "victim"
+ victim.write_text("SECRET")
+ (tmp_path / f"m.db{suffix}").symlink_to(victim)
+ with pytest.raises(PermissionError):
+ fileperms.connect_private_sqlite(db)
+ assert _mode(victim) == 0o644
+ assert victim.read_text() == "SECRET"
+
+ def test_read_only_uri_does_not_create_missing_db(self, tmp_path, monkeypatch):
+ monkeypatch.chdir(tmp_path)
+ with pytest.raises(sqlite3.OperationalError):
+ fileperms.connect_private_sqlite("file:missing.db?mode=ro", uri=True)
+ assert not (tmp_path / "missing.db").exists()
+
+ def test_uri_with_remote_authority_is_refused(self):
+ with pytest.raises(PermissionError):
+ fileperms.connect_private_sqlite("file://otherhost/srv/m.db", uri=True)
+
+ def test_refuses_symlinked_db(self, tmp_path):
+ target = tmp_path / "victim.db"
+ sqlite3.connect(target).close()
+ link = tmp_path / "m.db"
+ link.symlink_to(target)
+ with pytest.raises(PermissionError):
+ fileperms.connect_private_sqlite(link)
+
+
+class TestPrivateDir:
+ def test_new_and_existing_dirs_are_0700(self, tmp_path):
+ new = tmp_path / "a" / "memories"
+ fileperms.private_dir(new)
+ assert _mode(new) == 0o700
+ old = tmp_path / "old"
+ old.mkdir()
+ assert _mode(old) == 0o755
+ fileperms.private_dir(old)
+ assert _mode(old) == 0o700
+
+ def test_symlinked_dir_is_not_re_permissioned(self, tmp_path):
+ target = tmp_path / "target"
+ target.mkdir()
+ link = tmp_path / "memories"
+ link.symlink_to(target)
+ fileperms.private_dir(link)
+ assert _mode(target) == 0o755
+
+
+class TestOpenOwnerOnlyWriteMode:
+ def test_write_mode_truncates_and_is_private(self, tmp_path):
+ p = tmp_path / "cache.json"
+ p.write_text("old-and-long")
+ assert _mode(p) == 0o644
+ with fileperms.open_owner_only(p, "w") as fh:
+ fh.write("new")
+ assert p.read_text() == "new"
+ assert _mode(p) == 0o600
+
+ def test_append_mode_still_appends(self, tmp_path):
+ p = tmp_path / "log"
+ with fileperms.open_owner_only(p, "a") as fh:
+ fh.write("a")
+ with fileperms.open_owner_only(p, "a") as fh:
+ fh.write("b")
+ assert p.read_text() == "ab"
+
+ def test_read_modes_are_rejected(self, tmp_path):
+ with pytest.raises(ValueError):
+ fileperms.open_owner_only(tmp_path / "x", "r")
+
+
+# ---- call sites -------------------------------------------------------------
+
+
+class TestMemoryStoresArePrivate:
+ def test_sqlite_memory_store(self, tmp_path):
+ from headroom.memory.adapters.sqlite import SQLiteMemoryStore
+
+ p = tmp_path / "headroom_memory.db"
+ SQLiteMemoryStore(p)
+ assert _mode(p) == 0o600
+
+ def test_sqlite_memory_store_narrows_existing_db(self, tmp_path):
+ from headroom.memory.adapters.sqlite import SQLiteMemoryStore
+
+ p = tmp_path / "headroom_memory.db"
+ sqlite3.connect(p).close() # an older, unhardened run left it 0644
+ assert _mode(p) == 0o644
+ SQLiteMemoryStore(p)
+ assert _mode(p) == 0o600
+
+ def test_fts5_index(self, tmp_path):
+ from headroom.memory.adapters.fts5 import FTS5TextIndex
+
+ p = tmp_path / "headroom_memory.db"
+ try:
+ FTS5TextIndex(p)
+ except sqlite3.OperationalError as exc: # sqlite built without FTS5
+ pytest.skip(f"FTS5 unavailable: {exc}")
+ assert _mode(p) == 0o600
+
+ def test_graph_store(self, tmp_path):
+ from headroom.memory.adapters.sqlite_graph import SQLiteGraphStore
+
+ p = tmp_path / "headroom_graph.db"
+ SQLiteGraphStore(p)
+ assert _mode(p) == 0o600
+
+ def test_vector_index(self, tmp_path):
+ pytest.importorskip("sqlite_vec")
+ from headroom.memory.adapters.sqlite_vector import SQLiteVectorIndex
+
+ p = tmp_path / "vectors.db"
+ try:
+ SQLiteVectorIndex(dimension=8, db_path=p)
+ except (ImportError, RuntimeError) as exc:
+ pytest.skip(f"sqlite-vec unavailable here: {exc}")
+ assert _mode(p) == 0o600
+
+ @staticmethod
+ def _hnsw_index():
+ # Never ``importorskip("hnswlib")`` here: on runners without AVX the
+ # native import dies with SIGILL and takes the whole pytest shard with
+ # it. The adapter's probe runs the import in a subprocess.
+ import asyncio
+
+ from headroom.memory.adapters.hnsw import _check_hnswlib_available
+
+ if not _check_hnswlib_available():
+ pytest.skip("hnswlib not available (or not usable on this CPU)")
+
+ import numpy as np
+
+ from headroom.memory.adapters.hnsw import HNSWVectorIndex
+ from headroom.memory.models import Memory
+
+ index = HNSWVectorIndex(dimension=4, max_elements=16)
+ mem = Memory(content="secret fact", user_id="alice", embedding=np.ones(4, dtype=np.float32))
+ asyncio.run(index.index(mem))
+ return index
+
+ def test_hnsw_index_and_metadata_files(self, tmp_path):
+ index = self._hnsw_index()
+ base = tmp_path / "idx"
+ index.save_index(base)
+ assert _mode(base.with_suffix(".hnsw")) == 0o600
+ assert _mode(base.with_suffix(".meta")) == 0o600
+ # A re-save over a file left wide by an older version narrows it.
+ os.chmod(base.with_suffix(".hnsw"), 0o640)
+ index.save_index(base)
+ assert _mode(base.with_suffix(".hnsw")) == 0o600
+
+ def test_hnsw_index_refuses_planted_symlink(self, tmp_path):
+ index = self._hnsw_index()
+ target = tmp_path / "elsewhere.bin"
+ target.write_bytes(b"untouched")
+ base = tmp_path / "idx"
+ base.with_suffix(".hnsw").symlink_to(target)
+ with pytest.raises(PermissionError):
+ index.save_index(base)
+ assert target.read_bytes() == b"untouched"
+
+
+def test_native_memory_dir_is_0700(tmp_path):
+ from headroom.proxy.memory_handler import MemoryConfig, MemoryHandler
+
+ d = tmp_path / "memories"
+ d.mkdir()
+ assert _mode(d) == 0o755
+ handler = MemoryHandler(MemoryConfig(native_memory_dir=str(d)))
+ handler._init_native_memory_dir() # what initialize() runs for the native tool
+ assert _mode(d) == 0o700
+
+
+def test_license_cache_is_private(tmp_path):
+ from headroom.telemetry.reporter import LicenseInfo, UsageReporter
+
+ cache = tmp_path / "license.json"
+ cache.write_text("{}")
+ assert _mode(cache) == 0o644
+ reporter = UsageReporter("hlk_test", cache_path=cache)
+ reporter._license_info = LicenseInfo(status="active")
+ reporter._save_cache()
+ assert _mode(cache) == 0o600
+ assert "active" in cache.read_text()
+
+
+def test_ccr_backend_still_private_via_shared_primitive(tmp_path):
+ from headroom.cache.backends.sqlite import SQLiteBackend
+
+ p = tmp_path / "ccr_store.db"
+ SQLiteBackend(p)
+ assert _mode(p) == 0o600
diff --git a/tests/test_usage_reporter_snapshot.py b/tests/test_usage_reporter_snapshot.py
index 314ebded1..7cd130bbf 100644
--- a/tests/test_usage_reporter_snapshot.py
+++ b/tests/test_usage_reporter_snapshot.py
@@ -8,6 +8,7 @@ from __future__ import annotations
from types import SimpleNamespace
import anyio
+import pytest
from headroom.telemetry.reporter import UsageReporter
@@ -70,3 +71,28 @@ def test_baseline_not_advanced_on_exception():
anyio.run(r._report_usage)
assert r._last_tokens_saved_by_model == {}
assert r._last_requests_by_model == {}
+
+
+def test_license_cache_is_written_with_lf(tmp_path, monkeypatch):
+ """The cache is indented JSON; pin LF so Windows does not write CRLF (#3698).
+
+ Asserted at the call: on POSIX an unpinned write is identical on disk.
+ """
+ from headroom import fileperms
+ from headroom.telemetry.reporter import LicenseInfo
+
+ calls = []
+ original = fileperms.open_owner_only
+
+ def spy(path, mode="a", **kwargs):
+ calls.append(kwargs.get("newline"))
+ return original(path, mode, **kwargs)
+
+ monkeypatch.setattr(fileperms, "open_owner_only", spy)
+ cache = tmp_path / "license.json"
+ reporter = UsageReporter("hlk_test", cache_path=cache)
+ reporter._license_info = LicenseInfo(status="active")
+ reporter._save_cache()
+ assert calls == ["\n"]
+ assert b"\r\n" not in cache.read_bytes()