160-tool MCP server for real-time global intelligence from free public APIs: markets, liquidity, prediction markets, SEC filings, conflict, military, aircraft-cohort early warning, migration and border flows, cyber, climate, news and more. Dated observation history, 29 research prompts, cited briefs, geofences, SSE dashboard. MIT, no paid API keys.
Python ai-toolsai-watchanthropicclaudecybersecuritydashboardgeofencinggeopoliticalgeospatialintelligencemarketsmcpmigrationmilitarymodel-context-protocolnuclearosintreal-timespace-weatherthreat-intelligence
History: write the failure record through an exclusive temp file
Phoenix Agent (mac-studio)9db66a79fec0
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex ec41249..9f7af3e 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -71,7 +71,12 @@fails too, the lost cycle is appended to`<cache db>-history-write-failures.jsonl` beside the DB (newest 500kept), and collection health reports it with the sources it covered. A- history write failure never fails the cycle.+ history write failure never fails the cycle. The record is rewritten+ through an exclusive temp file (`mkstemp`: random name, mode 0600)+ renamed into place, and is never read or written through a symlink. A+ symlink, FIFO or other non-regular file at its path is not read: the+ collector replaces it (logged, and noted in the new record as+ `replaced_unread`), and health names one it finds in its data gaps.- Expected cadence for health is 300 s (the shipped plist interval);`WORLD_INTEL_HISTORY_CADENCE_S` and`WORLD_INTEL_HISTORY_CADENCES=source=seconds,...` override it.diff --git a/src/world_intel_mcp/history.py b/src/world_intel_mcp/history.pyindex aa2601b..d3dc5a3 100644--- a/src/world_intel_mcp/history.py+++ b/src/world_intel_mcp/history.py@@ -50,7 +50,9 @@ success only from a live capture, so a source served from stale cachereads ``stale`` and goes overdue on schedule. When a cycle's write failstwice (a locked or unwritable DB), the collector appends a record of thelost cycle to ``<db>-history-write-failures.jsonl`` beside the DB (newest-500 kept), which ``collection_health`` reports.+500 kept), which ``collection_health`` reports. That file is rewritten+through an exclusive temp file and never read or written through a+symlink.Registering an extractor (the public API)::@@ -120,10 +122,14 @@ Gaps / not covered:- A collector and a server that resolve different cache paths keepdifferent run logs; health then reads ``never_run`` for every source andsays where it looked.+- The lost-cycle record is read when it is a regular file, whoever owns+ it: in a DB directory another local user can write, that user can plant+ records there (as they can tamper with the DB itself)."""from __future__ import annotations+import errnoimport hashlibimport jsonimport logging@@ -131,6 +137,7 @@ import mathimport osimport reimport sqlite3+import statimport tempfileimport timeimport zlib@@ -645,6 +652,79 @@ def write_failures_path(db_path: Path | str) -> Path:return db_path.with_name(db_path.name + _WRITE_FAILURES_SUFFIX)+# open() flags for reading the lost-cycle record: never through a symlink,+# never blocking on a FIFO (0 where the OS has no such flag).+_O_NOFOLLOW = getattr(os, "O_NOFOLLOW", 0)+_O_NONBLOCK = getattr(os, "O_NONBLOCK", 0)+_SYMLINK_REFUSED = "is a symlink; not followed"+_NOT_REGULAR_REFUSED = "is not a regular file; not read"+++def _record_refusal(path: Path) -> str | None:+ """Why the lost-cycle record at ``path`` is not read, judged by the+ entry itself (``lstat``), or None when it is absent, a regular file, or+ cannot be inspected (opening it then raises that error)."""+ try:+ mode = os.lstat(path).st_mode+ except OSError:+ return None+ if stat.S_ISLNK(mode):+ return _SYMLINK_REFUSED+ if not stat.S_ISREG(mode):+ return _NOT_REGULAR_REFUSED+ return None+++def _read_record(path: Path) -> tuple[str | None, str | None]:+ """(text, None); (None, None) when there is no record yet; or+ (None, why) when it is refused.++ A symlink is never followed (an ``lstat`` check, and ``O_NOFOLLOW``+ where the OS has it in case the name is swapped after the check) and+ only a regular file is read, so a name planted beside the DB, in a+ directory another user can write, cannot point the read at another+ file or hang it on a FIFO. Undecodable bytes are replaced, so such a+ line counts as unreadable instead of raising.+ """+ refused = _record_refusal(path)+ if refused:+ return None, refused+ try:+ fd = os.open(path, os.O_RDONLY | _O_NOFOLLOW | _O_NONBLOCK)+ except FileNotFoundError:+ return None, None+ except OSError as exc:+ # O_NOFOLLOW on a symlink: ELOOP (Linux, macOS) or EMLINK (FreeBSD).+ if exc.errno in (errno.ELOOP, errno.EMLINK):+ return None, _SYMLINK_REFUSED+ raise+ with os.fdopen(fd, "rb") as fh:+ if not stat.S_ISREG(os.fstat(fh.fileno()).st_mode):+ return None, _NOT_REGULAR_REFUSED+ return fh.read().decode("utf-8", errors="replace"), None+++def _replace_privately(path: Path, text: str) -> None:+ """Write ``text`` to ``path`` through an exclusive temp file beside it.++ ``mkstemp`` creates the temp file with O_EXCL, an unpredictable name+ and mode 0600, and it is renamed over ``path``. ``path`` itself is+ never opened, so a symlink there is replaced, not written through. The+ temp file is removed on any error.+ """+ fd, tmp = tempfile.mkstemp(dir=str(path.parent), prefix=path.name + ".", suffix=".tmp")+ try:+ with os.fdopen(fd, "w", encoding="utf-8") as fh:+ fh.write(text)+ os.replace(tmp, path)+ except BaseException:+ try:+ os.unlink(tmp)+ except OSError as exc:+ logger.warning("Could not remove the temp file %s: %s", tmp, exc)+ raise++def record_write_failure(db_path: Path | str, outcomes: Iterable[SourceOutcome], error: str,now: float | None = None,@@ -652,9 +732,13 @@ def record_write_failure("""Keep a lasting record of a cycle whose history write failed.Written beside the DB, not in it (the DB may be what failed), newest- ``_WRITE_FAILURES_KEPT`` records kept. Returns the file, or None when- it could not be written either; that is logged, and the collector log- is then the only trace of the lost cycle.+ ``_WRITE_FAILURES_KEPT`` records kept, through an exclusive temp file+ renamed into place. The record is never read or written through a+ symlink: a symlink (or anything but a regular file) at its path is+ replaced unread, which is logged and noted in the new record as+ ``replaced_unread``. Returns the file, or None when it could not be+ written either; that is logged, and the collector log is then the only+ trace of the lost cycle."""now = time.time() if now is None else nowpath = write_failures_path(db_path)@@ -665,12 +749,13 @@ def record_write_failure("sources": sorted({outcome.source for outcome in outcomes}),}try:- lines = path.read_text().splitlines() if path.exists() else []+ text, refused = _read_record(path)+ if refused:+ logger.error("Not reading the lost-cycle record %s: it %s; replacing it", path, refused)+ record["replaced_unread"] = f"{path} {refused}"+ lines = (text or "").splitlines()lines = (lines + [json.dumps(record, separators=(",", ":"))])[-_WRITE_FAILURES_KEPT:]- partial = path.with_name(path.name + ".tmp")- partial.write_text("\n".join(lines) + "\n")- os.chmod(partial, 0o600)- os.replace(partial, path)+ _replace_privately(path, "\n".join(lines) + "\n")except OSError as exc:logger.error("Could not record the history write failure at %s: %s", path, exc)return None@@ -678,15 +763,22 @@ def record_write_failure(def read_write_failures(db_path: Path | str, now: float | None = None) -> tuple[list[dict], int]:- """(lost-cycle records inside the run-log retention window, unreadable count)."""+ """(lost-cycle records inside the run-log retention window, unreadable count).++ A record file that is a symlink or not a regular file is not read; it+ is logged and counts as one unreadable record.+ """path = write_failures_path(db_path)try:- text = path.read_text()- except FileNotFoundError:- return [], 0+ text, refused = _read_record(path)except OSError as exc:logger.warning("Could not read %s: %s", path, exc)return [], 1+ if refused:+ logger.warning("Not reading the lost-cycle record %s: it %s", path, refused)+ return [], 1+ if text is None:+ return [], 0keep_days = retention_days()["capture_days"]now = time.time() if now is None else nowcutoff = now - keep_days * 86400 if keep_days > 0 else None@@ -1150,8 +1242,13 @@ class HistoryStore:"can be off (unrecorded_attempts per source)")if unreadable_failures:+ refused = _record_refusal(record_file)gaps.append(- f"{unreadable_failures} lost-cycle record(s) in {record_file} could not be read"+ f"the lost-cycle record {record_file} {refused}, so cycles whose history "+ "write failed are not counted here"+ if refused+ else f"{unreadable_failures} lost-cycle record(s) in {record_file} could not "+ "be read")return {"sources": rows,diff --git a/src/world_intel_mcp/tests/test_history_honesty.py b/src/world_intel_mcp/tests/test_history_honesty.pyindex 42d1260..07dd8f6 100644--- a/src/world_intel_mcp/tests/test_history_honesty.py+++ b/src/world_intel_mcp/tests/test_history_honesty.py@@ -12,8 +12,11 @@ the collector attaches, at fixed times.import asyncioimport json+import osimport sqlite3+import statimport tempfile+import threadingimport timeimport typesfrom urllib.parse import unquote@@ -614,6 +617,136 @@ def test_unwritable_failure_record_is_logged_not_raised(tmp_path, caplog) -> Nonassert "Could not record the history write failure" in caplog.text+# The symlink and FIFO tests below need POSIX (symlinks, os.mkfifo,+# O_NOFOLLOW), as CI (Ubuntu) and macOS provide.+_LOST = [SourceOutcome("wildfires", T0)]+++def _without_hanging(fn, fifo):+ """(fn's result, whether it blocked). A reader blocked opening the FIFO+ is released by opening its write end, so a regression fails, not hangs."""+ out = {}+ thread = threading.Thread(target=lambda: out.update(result=fn()), daemon=True)+ thread.start()+ thread.join(5)+ blocked = thread.is_alive()+ if blocked:+ os.close(os.open(fifo, os.O_WRONLY | os.O_NONBLOCK))+ thread.join(5)+ return out.get("result"), blocked+++def test_failure_record_is_never_written_through_a_symlink(tmp_path, caplog) -> None:+ db = tmp_path / "cache.db"+ path = history.write_failures_path(db)+ targets = [tmp_path / "other-file-1", tmp_path / "other-file-2"]+ for target in targets:+ target.write_text("not ours\n")+ target.chmod(0o644)+ (tmp_path / (path.name + ".tmp")).symlink_to(targets[0]) # the old fixed temp name+ path.symlink_to(targets[1])++ assert history.record_write_failure(db, _LOST, "boom", now=T0) == path+ for target in targets:+ assert target.read_text() == "not ours\n"+ assert stat.S_IMODE(target.stat().st_mode) == 0o644+ assert not path.is_symlink() and stat.S_IMODE(path.stat().st_mode) == 0o600+ (line,) = path.read_text().splitlines()+ record = json.loads(line)+ assert (record["error"], record["replaced_unread"]) == (+ "boom", f"{path} is a symlink; not followed"+ )+ assert f"Not reading the lost-cycle record {path}: it is a symlink; not followed" in caplog.text+ # Nothing else was created: no temp file is left behind.+ assert sorted(p.name for p in tmp_path.iterdir()) == sorted(+ [path.name, path.name + ".tmp", "other-file-1", "other-file-2"]+ )+++def test_lost_cycle_reader_does_not_follow_a_symlink(tmp_path, caplog) -> None:+ db = tmp_path / "cache.db"+ path = history.write_failures_path(db)+ elsewhere = tmp_path / "elsewhere.jsonl"+ elsewhere.write_text(+ json.dumps({"failed_at_ts": T0, "error": "planted", "sources": ["wildfires"]}) + "\n"+ )+ path.symlink_to(elsewhere)+ assert history.read_write_failures(db, now=T0) == ([], 1)+ assert f"Not reading the lost-cycle record {path}: it is a symlink; not followed" in caplog.text+ s = HistoryStore(db)+ try:+ health = s.collection_health(["wildfires"], now=T0)+ finally:+ s.close()+ assert health["history_write_failures"]["count"] == 0+ assert (+ f"the lost-cycle record {path} is a symlink; not followed, so cycles whose history "+ "write failed are not counted here"+ ) in health["data_gaps"]+++def test_a_name_swapped_after_the_check_is_still_refused(tmp_path, monkeypatch) -> None:+ monkeypatch.setattr(history, "_record_refusal", lambda path: None) # the lstat check missed it+ (tmp_path / "elsewhere").write_text("x\n")+ link = tmp_path / "link.jsonl"+ link.symlink_to(tmp_path / "elsewhere")+ assert history._read_record(link) == (None, "is a symlink; not followed")+ fifo = tmp_path / "fifo.jsonl"+ os.mkfifo(fifo)+ result, blocked = _without_hanging(lambda: history._read_record(fifo), fifo)+ assert (result, blocked) == ((None, "is not a regular file; not read"), False)+++def test_a_fifo_at_the_record_path_is_refused_without_blocking(tmp_path) -> None:+ db = tmp_path / "cache.db"+ path = history.write_failures_path(db)+ os.mkfifo(path)+ result, blocked = _without_hanging(lambda: history.read_write_failures(db, now=T0), path)+ assert (result, blocked) == (([], 1), False)+ assert history.record_write_failure(db, _LOST, "boom", now=T0) == path+ assert json.loads(path.read_text())["replaced_unread"] == f"{path} is not a regular file; not read"+++def test_a_failed_record_write_leaves_no_temp_file(tmp_path, caplog) -> None:+ db = tmp_path / "cache.db"+ path = history.write_failures_path(db)+ path.mkdir()+ (path / "keep").write_text("x")+ assert history.record_write_failure(db, _LOST, "boom", now=T0) is None+ assert "Could not record the history write failure" in caplog.text+ assert [p.name for p in tmp_path.iterdir()] == [path.name]+ assert (path / "keep").read_text() == "x"+ s = HistoryStore(db)+ try:+ gaps = s.collection_health(["wildfires"], now=T0)["data_gaps"]+ finally:+ s.close()+ assert (+ f"the lost-cycle record {path} is not a regular file; not read, so cycles whose "+ "history write failed are not counted here"+ ) in gaps+++def test_undecodable_lost_cycle_lines_count_as_unreadable(tmp_path) -> None:+ db = tmp_path / "cache.db"+ history.record_write_failure(db, _LOST, "boom", now=T0)+ with history.write_failures_path(db).open("ab") as fh:+ fh.write(b"\xff\xfe not utf-8\n")+ records, unreadable = history.read_write_failures(db, now=T0)+ assert ([r["error"] for r in records], unreadable) == (["boom"], 1)+ assert history.record_write_failure(db, _LOST, "again", now=T0 + 1) is not None+++def test_health_reports_a_record_it_cannot_inspect(tmp_path) -> None:+ db = tmp_path / ("d" * 236 + ".db") # the record's file name is over NAME_MAX+ s = HistoryStore(db)+ try:+ gaps = s.collection_health(["wildfires"], now=T0)["data_gaps"]+ finally:+ s.close()+ assert f"1 lost-cycle record(s) in {history.write_failures_path(db)} could not be read" in gaps++# ---------------------------------------------------------------------------# Surviving mutants from the review# ---------------------------------------------------------------------------