Skip to content

160-tool MCP server for real-time global intelligence from free public APIs: markets, liquidity, prediction markets, SEC filings, conflict, military, aircraft-cohort early warning, migration and border flows, cyber, climate, news and more. Dated observation history, 29 research prompts, cited briefs, geofences, SSE dashboard. MIT, no paid API keys.

History: write the failure record through an exclusive temp file

Phoenix Agent (mac-studio)9db66a79fec0
diff --git a/CHANGELOG.md b/CHANGELOG.md
index ec41249..9f7af3e 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -71,7 +71,12 @@
fails too, the lost cycle is appended to
`<cache db>-history-write-failures.jsonl` beside the DB (newest 500
kept), and collection health reports it with the sources it covered. A
- history write failure never fails the cycle.
+ history write failure never fails the cycle. The record is rewritten
+ through an exclusive temp file (`mkstemp`: random name, mode 0600)
+ renamed into place, and is never read or written through a symlink. A
+ symlink, FIFO or other non-regular file at its path is not read: the
+ collector replaces it (logged, and noted in the new record as
+ `replaced_unread`), and health names one it finds in its data gaps.
- Expected cadence for health is 300 s (the shipped plist interval);
`WORLD_INTEL_HISTORY_CADENCE_S` and
`WORLD_INTEL_HISTORY_CADENCES=source=seconds,...` override it.
diff --git a/src/world_intel_mcp/history.py b/src/world_intel_mcp/history.py
index aa2601b..d3dc5a3 100644
--- a/src/world_intel_mcp/history.py
+++ b/src/world_intel_mcp/history.py
@@ -50,7 +50,9 @@ success only from a live capture, so a source served from stale cache
reads ``stale`` and goes overdue on schedule. When a cycle's write fails
twice (a locked or unwritable DB), the collector appends a record of the
lost cycle to ``<db>-history-write-failures.jsonl`` beside the DB (newest
-500 kept), which ``collection_health`` reports.
+500 kept), which ``collection_health`` reports. That file is rewritten
+through an exclusive temp file and never read or written through a
+symlink.
Registering an extractor (the public API)::
@@ -120,10 +122,14 @@ Gaps / not covered:
- A collector and a server that resolve different cache paths keep
different run logs; health then reads ``never_run`` for every source and
says where it looked.
+- The lost-cycle record is read when it is a regular file, whoever owns
+ it: in a DB directory another local user can write, that user can plant
+ records there (as they can tamper with the DB itself).
"""
from __future__ import annotations
+import errno
import hashlib
import json
import logging
@@ -131,6 +137,7 @@ import math
import os
import re
import sqlite3
+import stat
import tempfile
import time
import zlib
@@ -645,6 +652,79 @@ def write_failures_path(db_path: Path | str) -> Path:
return db_path.with_name(db_path.name + _WRITE_FAILURES_SUFFIX)
+# open() flags for reading the lost-cycle record: never through a symlink,
+# never blocking on a FIFO (0 where the OS has no such flag).
+_O_NOFOLLOW = getattr(os, "O_NOFOLLOW", 0)
+_O_NONBLOCK = getattr(os, "O_NONBLOCK", 0)
+_SYMLINK_REFUSED = "is a symlink; not followed"
+_NOT_REGULAR_REFUSED = "is not a regular file; not read"
+
+
+def _record_refusal(path: Path) -> str | None:
+ """Why the lost-cycle record at ``path`` is not read, judged by the
+ entry itself (``lstat``), or None when it is absent, a regular file, or
+ cannot be inspected (opening it then raises that error)."""
+ try:
+ mode = os.lstat(path).st_mode
+ except OSError:
+ return None
+ if stat.S_ISLNK(mode):
+ return _SYMLINK_REFUSED
+ if not stat.S_ISREG(mode):
+ return _NOT_REGULAR_REFUSED
+ return None
+
+
+def _read_record(path: Path) -> tuple[str | None, str | None]:
+ """(text, None); (None, None) when there is no record yet; or
+ (None, why) when it is refused.
+
+ A symlink is never followed (an ``lstat`` check, and ``O_NOFOLLOW``
+ where the OS has it in case the name is swapped after the check) and
+ only a regular file is read, so a name planted beside the DB, in a
+ directory another user can write, cannot point the read at another
+ file or hang it on a FIFO. Undecodable bytes are replaced, so such a
+ line counts as unreadable instead of raising.
+ """
+ refused = _record_refusal(path)
+ if refused:
+ return None, refused
+ try:
+ fd = os.open(path, os.O_RDONLY | _O_NOFOLLOW | _O_NONBLOCK)
+ except FileNotFoundError:
+ return None, None
+ except OSError as exc:
+ # O_NOFOLLOW on a symlink: ELOOP (Linux, macOS) or EMLINK (FreeBSD).
+ if exc.errno in (errno.ELOOP, errno.EMLINK):
+ return None, _SYMLINK_REFUSED
+ raise
+ with os.fdopen(fd, "rb") as fh:
+ if not stat.S_ISREG(os.fstat(fh.fileno()).st_mode):
+ return None, _NOT_REGULAR_REFUSED
+ return fh.read().decode("utf-8", errors="replace"), None
+
+
+def _replace_privately(path: Path, text: str) -> None:
+ """Write ``text`` to ``path`` through an exclusive temp file beside it.
+
+ ``mkstemp`` creates the temp file with O_EXCL, an unpredictable name
+ and mode 0600, and it is renamed over ``path``. ``path`` itself is
+ never opened, so a symlink there is replaced, not written through. The
+ temp file is removed on any error.
+ """
+ fd, tmp = tempfile.mkstemp(dir=str(path.parent), prefix=path.name + ".", suffix=".tmp")
+ try:
+ with os.fdopen(fd, "w", encoding="utf-8") as fh:
+ fh.write(text)
+ os.replace(tmp, path)
+ except BaseException:
+ try:
+ os.unlink(tmp)
+ except OSError as exc:
+ logger.warning("Could not remove the temp file %s: %s", tmp, exc)
+ raise
+
+
def record_write_failure(
db_path: Path | str, outcomes: Iterable[SourceOutcome], error: str,
now: float | None = None,
@@ -652,9 +732,13 @@ def record_write_failure(
"""Keep a lasting record of a cycle whose history write failed.
Written beside the DB, not in it (the DB may be what failed), newest
- ``_WRITE_FAILURES_KEPT`` records kept. Returns the file, or None when
- it could not be written either; that is logged, and the collector log
- is then the only trace of the lost cycle.
+ ``_WRITE_FAILURES_KEPT`` records kept, through an exclusive temp file
+ renamed into place. The record is never read or written through a
+ symlink: a symlink (or anything but a regular file) at its path is
+ replaced unread, which is logged and noted in the new record as
+ ``replaced_unread``. Returns the file, or None when it could not be
+ written either; that is logged, and the collector log is then the only
+ trace of the lost cycle.
"""
now = time.time() if now is None else now
path = write_failures_path(db_path)
@@ -665,12 +749,13 @@ def record_write_failure(
"sources": sorted({outcome.source for outcome in outcomes}),
}
try:
- lines = path.read_text().splitlines() if path.exists() else []
+ text, refused = _read_record(path)
+ if refused:
+ logger.error("Not reading the lost-cycle record %s: it %s; replacing it", path, refused)
+ record["replaced_unread"] = f"{path} {refused}"
+ lines = (text or "").splitlines()
lines = (lines + [json.dumps(record, separators=(",", ":"))])[-_WRITE_FAILURES_KEPT:]
- partial = path.with_name(path.name + ".tmp")
- partial.write_text("\n".join(lines) + "\n")
- os.chmod(partial, 0o600)
- os.replace(partial, path)
+ _replace_privately(path, "\n".join(lines) + "\n")
except OSError as exc:
logger.error("Could not record the history write failure at %s: %s", path, exc)
return None
@@ -678,15 +763,22 @@ def record_write_failure(
def read_write_failures(db_path: Path | str, now: float | None = None) -> tuple[list[dict], int]:
- """(lost-cycle records inside the run-log retention window, unreadable count)."""
+ """(lost-cycle records inside the run-log retention window, unreadable count).
+
+ A record file that is a symlink or not a regular file is not read; it
+ is logged and counts as one unreadable record.
+ """
path = write_failures_path(db_path)
try:
- text = path.read_text()
- except FileNotFoundError:
- return [], 0
+ text, refused = _read_record(path)
except OSError as exc:
logger.warning("Could not read %s: %s", path, exc)
return [], 1
+ if refused:
+ logger.warning("Not reading the lost-cycle record %s: it %s", path, refused)
+ return [], 1
+ if text is None:
+ return [], 0
keep_days = retention_days()["capture_days"]
now = time.time() if now is None else now
cutoff = now - keep_days * 86400 if keep_days > 0 else None
@@ -1150,8 +1242,13 @@ class HistoryStore:
"can be off (unrecorded_attempts per source)"
)
if unreadable_failures:
+ refused = _record_refusal(record_file)
gaps.append(
- f"{unreadable_failures} lost-cycle record(s) in {record_file} could not be read"
+ f"the lost-cycle record {record_file} {refused}, so cycles whose history "
+ "write failed are not counted here"
+ if refused
+ else f"{unreadable_failures} lost-cycle record(s) in {record_file} could not "
+ "be read"
)
return {
"sources": rows,
diff --git a/src/world_intel_mcp/tests/test_history_honesty.py b/src/world_intel_mcp/tests/test_history_honesty.py
index 42d1260..07dd8f6 100644
--- a/src/world_intel_mcp/tests/test_history_honesty.py
+++ b/src/world_intel_mcp/tests/test_history_honesty.py
@@ -12,8 +12,11 @@ the collector attaches, at fixed times.
import asyncio
import json
+import os
import sqlite3
+import stat
import tempfile
+import threading
import time
import types
from urllib.parse import unquote
@@ -614,6 +617,136 @@ def test_unwritable_failure_record_is_logged_not_raised(tmp_path, caplog) -> Non
assert "Could not record the history write failure" in caplog.text
+# The symlink and FIFO tests below need POSIX (symlinks, os.mkfifo,
+# O_NOFOLLOW), as CI (Ubuntu) and macOS provide.
+_LOST = [SourceOutcome("wildfires", T0)]
+
+
+def _without_hanging(fn, fifo):
+ """(fn's result, whether it blocked). A reader blocked opening the FIFO
+ is released by opening its write end, so a regression fails, not hangs."""
+ out = {}
+ thread = threading.Thread(target=lambda: out.update(result=fn()), daemon=True)
+ thread.start()
+ thread.join(5)
+ blocked = thread.is_alive()
+ if blocked:
+ os.close(os.open(fifo, os.O_WRONLY | os.O_NONBLOCK))
+ thread.join(5)
+ return out.get("result"), blocked
+
+
+def test_failure_record_is_never_written_through_a_symlink(tmp_path, caplog) -> None:
+ db = tmp_path / "cache.db"
+ path = history.write_failures_path(db)
+ targets = [tmp_path / "other-file-1", tmp_path / "other-file-2"]
+ for target in targets:
+ target.write_text("not ours\n")
+ target.chmod(0o644)
+ (tmp_path / (path.name + ".tmp")).symlink_to(targets[0]) # the old fixed temp name
+ path.symlink_to(targets[1])
+
+ assert history.record_write_failure(db, _LOST, "boom", now=T0) == path
+ for target in targets:
+ assert target.read_text() == "not ours\n"
+ assert stat.S_IMODE(target.stat().st_mode) == 0o644
+ assert not path.is_symlink() and stat.S_IMODE(path.stat().st_mode) == 0o600
+ (line,) = path.read_text().splitlines()
+ record = json.loads(line)
+ assert (record["error"], record["replaced_unread"]) == (
+ "boom", f"{path} is a symlink; not followed"
+ )
+ assert f"Not reading the lost-cycle record {path}: it is a symlink; not followed" in caplog.text
+ # Nothing else was created: no temp file is left behind.
+ assert sorted(p.name for p in tmp_path.iterdir()) == sorted(
+ [path.name, path.name + ".tmp", "other-file-1", "other-file-2"]
+ )
+
+
+def test_lost_cycle_reader_does_not_follow_a_symlink(tmp_path, caplog) -> None:
+ db = tmp_path / "cache.db"
+ path = history.write_failures_path(db)
+ elsewhere = tmp_path / "elsewhere.jsonl"
+ elsewhere.write_text(
+ json.dumps({"failed_at_ts": T0, "error": "planted", "sources": ["wildfires"]}) + "\n"
+ )
+ path.symlink_to(elsewhere)
+ assert history.read_write_failures(db, now=T0) == ([], 1)
+ assert f"Not reading the lost-cycle record {path}: it is a symlink; not followed" in caplog.text
+ s = HistoryStore(db)
+ try:
+ health = s.collection_health(["wildfires"], now=T0)
+ finally:
+ s.close()
+ assert health["history_write_failures"]["count"] == 0
+ assert (
+ f"the lost-cycle record {path} is a symlink; not followed, so cycles whose history "
+ "write failed are not counted here"
+ ) in health["data_gaps"]
+
+
+def test_a_name_swapped_after_the_check_is_still_refused(tmp_path, monkeypatch) -> None:
+ monkeypatch.setattr(history, "_record_refusal", lambda path: None) # the lstat check missed it
+ (tmp_path / "elsewhere").write_text("x\n")
+ link = tmp_path / "link.jsonl"
+ link.symlink_to(tmp_path / "elsewhere")
+ assert history._read_record(link) == (None, "is a symlink; not followed")
+ fifo = tmp_path / "fifo.jsonl"
+ os.mkfifo(fifo)
+ result, blocked = _without_hanging(lambda: history._read_record(fifo), fifo)
+ assert (result, blocked) == ((None, "is not a regular file; not read"), False)
+
+
+def test_a_fifo_at_the_record_path_is_refused_without_blocking(tmp_path) -> None:
+ db = tmp_path / "cache.db"
+ path = history.write_failures_path(db)
+ os.mkfifo(path)
+ result, blocked = _without_hanging(lambda: history.read_write_failures(db, now=T0), path)
+ assert (result, blocked) == (([], 1), False)
+ assert history.record_write_failure(db, _LOST, "boom", now=T0) == path
+ assert json.loads(path.read_text())["replaced_unread"] == f"{path} is not a regular file; not read"
+
+
+def test_a_failed_record_write_leaves_no_temp_file(tmp_path, caplog) -> None:
+ db = tmp_path / "cache.db"
+ path = history.write_failures_path(db)
+ path.mkdir()
+ (path / "keep").write_text("x")
+ assert history.record_write_failure(db, _LOST, "boom", now=T0) is None
+ assert "Could not record the history write failure" in caplog.text
+ assert [p.name for p in tmp_path.iterdir()] == [path.name]
+ assert (path / "keep").read_text() == "x"
+ s = HistoryStore(db)
+ try:
+ gaps = s.collection_health(["wildfires"], now=T0)["data_gaps"]
+ finally:
+ s.close()
+ assert (
+ f"the lost-cycle record {path} is not a regular file; not read, so cycles whose "
+ "history write failed are not counted here"
+ ) in gaps
+
+
+def test_undecodable_lost_cycle_lines_count_as_unreadable(tmp_path) -> None:
+ db = tmp_path / "cache.db"
+ history.record_write_failure(db, _LOST, "boom", now=T0)
+ with history.write_failures_path(db).open("ab") as fh:
+ fh.write(b"\xff\xfe not utf-8\n")
+ records, unreadable = history.read_write_failures(db, now=T0)
+ assert ([r["error"] for r in records], unreadable) == (["boom"], 1)
+ assert history.record_write_failure(db, _LOST, "again", now=T0 + 1) is not None
+
+
+def test_health_reports_a_record_it_cannot_inspect(tmp_path) -> None:
+ db = tmp_path / ("d" * 236 + ".db") # the record's file name is over NAME_MAX
+ s = HistoryStore(db)
+ try:
+ gaps = s.collection_health(["wildfires"], now=T0)["data_gaps"]
+ finally:
+ s.close()
+ assert f"1 lost-cycle record(s) in {history.write_failures_path(db)} could not be read" in gaps
+
+
# ---------------------------------------------------------------------------
# Surviving mutants from the review
# ---------------------------------------------------------------------------