160-tool MCP server for real-time global intelligence from free public APIs: markets, liquidity, prediction markets, SEC filings, conflict, military, aircraft-cohort early warning, migration and border flows, cyber, climate, news and more. Dated observation history, 29 research prompts, cited briefs, geofences, SSE dashboard. MIT, no paid API keys.
Python ai-toolsai-watchanthropicclaudecybersecuritydashboardgeofencinggeopoliticalgeospatialintelligencemarketsmcpmigrationmilitarymodel-context-protocolnuclearosintreal-timespace-weatherthreat-intelligence
Retry the WAL switch when simultaneous first opens collide
Phoenix Agent (mac-studio)4cbd5c9c82a4
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 2731e94..1c48ebb 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -100,6 +100,16 @@default 1,024 open-file limit. Test runs also no longer leave`wim-server-test-*` directories in the system temp directory.+### Fixed: simultaneous first opens of a cache database+- Processes opening a new cache database at the same moment (the MCP+ server, the collector and a CLI run, or parallel tests) could crash with+ "database is locked": the switch to WAL needs an exclusive lock, and+ SQLite refuses all but one of the simultaneous requests without waiting.+ A cache at the default path then gave up on that path and fell back to+ the temp directory. Opening now retries the switch for up to 10 seconds.+ Measured with 8 processes on one fresh file: 29 to 67 failed opens in 80+ to 160 before, none in 160 after.+### Fixed: the collector cycle summary separates data from errors- A source that answers with an upstream error (`{"error": ...}`) stillcounts as a completed fetch, but the cycle summary now lists it underdiff --git a/src/world_intel_mcp/cache.py b/src/world_intel_mcp/cache.pyindex 47fd25f..ab8b99b 100644--- a/src/world_intel_mcp/cache.py+++ b/src/world_intel_mcp/cache.py@@ -16,6 +16,10 @@ from typing import Anylogger = logging.getLogger("world-intel-mcp.cache")+# How long opening a connection keeps retrying the switch to WAL while+# another connection holds the database. Matches the connect timeout.+_WAL_RETRY_SECONDS = 10.0+def _default_db_path() -> Path:# Resolved per Cache, not at import, so HOME and XDG_CACHE_HOME are read@@ -77,6 +81,29 @@ def _fallback_db_path() -> Path:)+def _enable_wal(conn: sqlite3.Connection) -> None:+ """Switch the connection's database to WAL, waiting out other openers.++ Leaving rollback-journal mode needs an exclusive lock. When several+ processes open a new database at once, each holds a shared lock while+ asking for that exclusive one, and SQLite answers all but one with+ "database is locked" at once, without its busy handler, to avoid a+ deadlock. Retrying after a short pause lets the winner finish; any+ other error, or a lock still held after _WAL_RETRY_SECONDS, is raised.+ """+ deadline = time.monotonic() + _WAL_RETRY_SECONDS+ delay = 0.01+ while True:+ try:+ conn.execute("PRAGMA journal_mode=WAL")+ return+ except sqlite3.OperationalError as exc:+ if "locked" not in str(exc) or time.monotonic() >= deadline:+ raise+ time.sleep(delay)+ delay = min(delay * 2, 0.25)++class Cache:"""SQLite-backed TTL cache."""@@ -138,7 +165,7 @@ class Cache:def _get_conn(self) -> sqlite3.Connection:if self._conn is None:self._conn = sqlite3.connect(str(self.db_path), timeout=10)- self._conn.execute("PRAGMA journal_mode=WAL")+ _enable_wal(self._conn)self._conn.execute("PRAGMA busy_timeout=5000")self._conn.execute("PRAGMA synchronous=NORMAL")return self._conndiff --git a/src/world_intel_mcp/tests/test_cache_wal_init.py b/src/world_intel_mcp/tests/test_cache_wal_init.pynew file mode 100644index 0000000..4c4013a--- /dev/null+++ b/src/world_intel_mcp/tests/test_cache_wal_init.py@@ -0,0 +1,103 @@+"""Opening a new cache database from several connections at once.++The switch to WAL needs an exclusive lock, and SQLite answers the losers+of a simultaneous first open with "database is locked" without waiting.+Before the retry, 8 processes opening one fresh file crashed 29 to 67+times in 80 to 160 opens, and the default-path cache treated that as an+unusable path and fell back to the temp dir.++Gaps / not covered: the stress test uses threads in one process, which+hit the race less often than separate processes do (measured about 3 to+4 failed opens per 1,600 with the old code, against 67 per 160 for 8+processes); a lock held past the retry window is tested with a shortened+window, not the real 10 s.+"""++import pathlib+import sqlite3+import threading++import pytest++from world_intel_mcp import cache as cache_mod+from world_intel_mcp.cache import Cache+++class _LockedSwitch(sqlite3.Connection):+ """A connection whose first WAL switches report a held lock."""++ failures = 0+ message = "database is locked"++ def execute(self, sql, *args):+ if sql == "PRAGMA journal_mode=WAL" and type(self).failures > 0:+ type(self).failures -= 1+ raise sqlite3.OperationalError(type(self).message)+ return super().execute(sql, *args)+++def locked_connect(monkeypatch):+ real_connect = sqlite3.connect++ def connect(*args, **kwargs):+ return real_connect(*args, factory=_LockedSwitch, **kwargs)++ monkeypatch.setattr(cache_mod.sqlite3, "connect", connect)+ monkeypatch.setattr(cache_mod.time, "sleep", lambda _s: None)+ _LockedSwitch.message = "database is locked"+ yield _LockedSwitch+ _LockedSwitch.failures = 0+++def test_wal_switch_retries_a_held_lock(tmp_path, locked_connect):+ locked_connect.failures = 3+ c = Cache(db_path=tmp_path / "c.db")+ try:+ assert locked_connect.failures == 0+ assert c.fell_back is False+ assert c._get_conn().execute("PRAGMA journal_mode").fetchone()[0] == "wal"+ c.set("k", {"v": 1}, 60)+ assert c.get("k") == {"v": 1}+ finally:+ c.close()+++def test_wal_switch_raises_other_errors_at_once(tmp_path, locked_connect):+ locked_connect.failures = 1+ locked_connect.message = "disk I/O error"+ with pytest.raises(sqlite3.OperationalError, match="disk I/O error"):+ Cache(db_path=tmp_path / "c.db")+ assert locked_connect.failures == 0+++def test_wal_switch_gives_up_when_the_lock_outlasts_the_window(+ tmp_path, locked_connect, monkeypatch+):+ monkeypatch.setattr(cache_mod, "_WAL_RETRY_SECONDS", 0.0)+ locked_connect.failures = 1_000+ with pytest.raises(sqlite3.OperationalError, match="locked"):+ Cache(db_path=tmp_path / "c.db")+ assert locked_connect.failures == 999+++def test_simultaneous_first_opens_all_succeed(tmp_path):+ threads_per_round, rounds = 16, 200+ errors: list[BaseException] = []+ for r in range(rounds):+ path = pathlib.Path(tmp_path / f"round-{r}.db")+ barrier = threading.Barrier(threads_per_round)++ def open_once() -> None:+ barrier.wait()+ try:+ Cache(db_path=path).close()+ except Exception as exc: # collected and asserted below+ errors.append(exc)++ threads = [threading.Thread(target=open_once) for _ in range(threads_per_round)]+ for t in threads:+ t.start()+ for t in threads:+ t.join()+ assert errors == []