Skip to content

160-tool MCP server for real-time global intelligence from free public APIs: markets, liquidity, prediction markets, SEC filings, conflict, military, aircraft-cohort early warning, migration and border flows, cyber, climate, news and more. Dated observation history, 29 research prompts, cited briefs, geofences, SSE dashboard. MIT, no paid API keys.

Isolate baseline databases; space every fetch retry per host

Phoenix Agent (mac-studio)89f4f8a6d65c
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 6311734..c0f092d 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,5 +1,17 @@
# Changelog
+## Unreleased
+
+- Temporal and keyword spike databases now follow `XDG_CACHE_HOME` like
+ the cache: `$XDG_CACHE_HOME/world-intel-mcp/temporal.db` and
+ `$XDG_CACHE_HOME/world-intel-mcp/keyword_spikes.db`. With it unset, they
+ remain in `~/.cache/world-intel-mcp/`. Users who set `XDG_CACHE_HOME`
+ can keep existing baselines by moving these two files from
+ `~/.cache/world-intel-mcp/` to `$XDG_CACHE_HOME/world-intel-mcp/`.
+ Importing the package no longer opens databases; they open on first use.
+- Apply per-source request spacing to every HTTP retry, including JSON,
+ text, XML and binary fetches.
+
## 0.11.0 - 2026-10-12
Cross-border migration (MCP tools, CLI, collector and a dashboard
diff --git a/README.md b/README.md
index 77f7474..6b9d7f6 100644
--- a/README.md
+++ b/README.md
@@ -574,11 +574,20 @@ built-in hotspots, but scoped to your AOI's own radius instead of a fixed
2-degree window.
AOIs persist in a dedicated table inside the same SQLite cache database
-the server already uses (`~/.cache/world-intel-mcp/cache.db` by default,
-or `$WORLD_INTEL_CACHE_DB`), so a scheduled agent can watch any named
+the server already uses (`$WORLD_INTEL_CACHE_DB` when set, otherwise
+`$XDG_CACHE_HOME/world-intel-mcp/cache.db`, falling back to
+`~/.cache/world-intel-mcp/cache.db`), so a scheduled agent can watch any named
area across restarts with `intel_aoi_list` / `intel_aoi_delete` to manage
them.
+Temporal and keyword spike baselines use `temporal.db` and
+`keyword_spikes.db` in `$XDG_CACHE_HOME/world-intel-mcp/`, or
+`~/.cache/world-intel-mcp/` when `XDG_CACHE_HOME` is unset.
+`WORLD_INTEL_CACHE_DB` only overrides the main cache database, not these
+baseline files. Databases open on first use, not on import. If you set
+`XDG_CACHE_HOME`, move the two baseline files from `~/.cache/world-intel-mcp/`
+to `$XDG_CACHE_HOME/world-intel-mcp/` to keep existing baselines.
+
---
## Vector Store
diff --git a/conftest.py b/conftest.py
new file mode 100644
index 0000000..ab02082
--- /dev/null
+++ b/conftest.py
@@ -0,0 +1,16 @@
+"""Isolate cache and temporary files before importing any package modules."""
+
+import os
+import tempfile
+from pathlib import Path
+
+_test_directory = tempfile.TemporaryDirectory(
+ prefix=".test-cache-", dir=Path(__file__).parent
+)
+_test_root = Path(_test_directory.name)
+(_test_root / "home").mkdir()
+os.environ["HOME"] = str(_test_root / "home")
+os.environ["XDG_CACHE_HOME"] = str(_test_root / "cache")
+os.environ["WORLD_INTEL_CACHE_DB"] = str(_test_root / "cache.db")
+os.environ["TMPDIR"] = str(_test_root)
+tempfile.tempdir = str(_test_root)
diff --git a/src/world_intel_mcp/analysis/alerts.py b/src/world_intel_mcp/analysis/alerts.py
index 173acfc..d61007a 100644
--- a/src/world_intel_mcp/analysis/alerts.py
+++ b/src/world_intel_mcp/analysis/alerts.py
@@ -247,7 +247,7 @@ async def fetch_weekly_trends(fetcher) -> dict:
Returns:
Dict with trends, trend_count, analysis_period, source.
"""
- from ..analysis.temporal import TemporalBaseline, _DB_PATH
+ from ..analysis.temporal import _default_db_path
from ..sources import intelligence
now = datetime.now(timezone.utc)
@@ -261,7 +261,7 @@ async def fetch_weekly_trends(fetcher) -> dict:
trends: list[dict] = []
try:
- conn = sqlite3.connect(_DB_PATH)
+ conn = sqlite3.connect(_default_db_path())
rows = conn.execute(
"SELECT key, count, mean, m2, updated_at FROM baselines WHERE count >= 5"
).fetchall()
diff --git a/src/world_intel_mcp/analysis/spikes.py b/src/world_intel_mcp/analysis/spikes.py
index 5c78e38..9937c5a 100644
--- a/src/world_intel_mcp/analysis/spikes.py
+++ b/src/world_intel_mcp/analysis/spikes.py
@@ -14,9 +14,15 @@ from datetime import datetime, timezone
logger = logging.getLogger("world-intel-mcp.analysis.spikes")
-_DB_PATH = os.path.join(
- os.path.expanduser("~"), ".cache", "world-intel-mcp", "keyword_spikes.db"
-)
+_DB_PATH: str | None = None
+
+
+def _default_db_path() -> str:
+ if _DB_PATH is not None:
+ return _DB_PATH
+ cache_home = os.environ.get("XDG_CACHE_HOME") or "~/.cache"
+ return os.path.join(os.path.expanduser(cache_home), "world-intel-mcp", "keyword_spikes.db")
+
_CVE_RE = re.compile(r"CVE-\d{4}-\d{4,}", re.IGNORECASE)
_APT_RE = re.compile(
@@ -30,9 +36,17 @@ _APT_RE = re.compile(
class KeywordSpikeDetector:
"""Detects keyword frequency spikes against rolling baselines."""
- def __init__(self, db_path: str = _DB_PATH):
- os.makedirs(os.path.dirname(db_path), exist_ok=True)
- self._conn = sqlite3.connect(db_path)
+ def __init__(self, db_path: str | None = None):
+ self._db_path = db_path
+ self._conn: sqlite3.Connection | None = None
+
+ def _get_conn(self) -> sqlite3.Connection:
+ if self._conn is not None:
+ return self._conn
+ if self._db_path is None:
+ self._db_path = _default_db_path()
+ os.makedirs(os.path.dirname(self._db_path), exist_ok=True)
+ self._conn = sqlite3.connect(self._db_path)
self._conn.execute(
"""CREATE TABLE IF NOT EXISTS kw_baselines (
keyword TEXT PRIMARY KEY,
@@ -43,10 +57,11 @@ class KeywordSpikeDetector:
)"""
)
self._conn.commit()
+ return self._conn
def _get_baseline(self, keyword: str) -> tuple[int, float, float]:
"""Get (count, mean, m2) for a keyword."""
- row = self._conn.execute(
+ row = self._get_conn().execute(
"SELECT count, mean, m2 FROM kw_baselines WHERE keyword = ?",
(keyword,),
).fetchone()
@@ -61,14 +76,15 @@ class KeywordSpikeDetector:
delta2 = value - mean
m2 += delta * delta2
now = datetime.now(timezone.utc).isoformat()
- self._conn.execute(
+ conn = self._get_conn()
+ conn.execute(
"""INSERT INTO kw_baselines (keyword, count, mean, m2, updated_at)
VALUES (?, ?, ?, ?, ?)
ON CONFLICT(keyword) DO UPDATE SET
count=?, mean=?, m2=?, updated_at=?""",
(keyword, n, mean, m2, now, n, mean, m2, now),
)
- self._conn.commit()
+ conn.commit()
def detect_spikes(
self,
diff --git a/src/world_intel_mcp/analysis/temporal.py b/src/world_intel_mcp/analysis/temporal.py
index 7420f51..f1ef57e 100644
--- a/src/world_intel_mcp/analysis/temporal.py
+++ b/src/world_intel_mcp/analysis/temporal.py
@@ -15,18 +15,30 @@ from datetime import datetime, timezone
logger = logging.getLogger("world-intel-mcp.analysis.temporal")
-_DB_PATH = os.path.join(
- os.path.expanduser("~"), ".cache", "world-intel-mcp", "temporal.db"
-)
+_DB_PATH: str | None = None
+
+
+def _default_db_path() -> str:
+ if _DB_PATH is not None:
+ return _DB_PATH
+ cache_home = os.environ.get("XDG_CACHE_HOME") or "~/.cache"
+ return os.path.join(os.path.expanduser(cache_home), "world-intel-mcp", "temporal.db")
class TemporalBaseline:
"""Streaming anomaly detector with Welford's algorithm + SQLite persistence."""
- def __init__(self, db_path: str = _DB_PATH):
+ def __init__(self, db_path: str | None = None):
self._db_path = db_path
- os.makedirs(os.path.dirname(db_path), exist_ok=True)
- self._conn = sqlite3.connect(db_path)
+ self._conn: sqlite3.Connection | None = None
+
+ def _get_conn(self) -> sqlite3.Connection:
+ if self._conn is not None:
+ return self._conn
+ if self._db_path is None:
+ self._db_path = _default_db_path()
+ os.makedirs(os.path.dirname(self._db_path), exist_ok=True)
+ self._conn = sqlite3.connect(self._db_path)
self._conn.execute(
"""CREATE TABLE IF NOT EXISTS baselines (
key TEXT PRIMARY KEY,
@@ -37,6 +49,7 @@ class TemporalBaseline:
)"""
)
self._conn.commit()
+ return self._conn
def _key(self, event_type: str, region: str) -> str:
"""Build a composite key including weekday and month for seasonality."""
@@ -49,8 +62,9 @@ class TemporalBaseline:
"""Record an observation using Welford's online update."""
key = self._key(event_type, region)
now_iso = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
+ conn = self._get_conn()
- row = self._conn.execute(
+ row = conn.execute(
"SELECT count, mean, m2 FROM baselines WHERE key = ?", (key,)
).fetchone()
@@ -66,7 +80,7 @@ class TemporalBaseline:
delta2 = count - mean
m2 += delta * delta2
- self._conn.execute(
+ conn.execute(
"""INSERT INTO baselines (key, count, mean, m2, updated_at)
VALUES (?, ?, ?, ?, ?)
ON CONFLICT(key) DO UPDATE SET
@@ -76,7 +90,7 @@ class TemporalBaseline:
updated_at = excluded.updated_at""",
(key, n, mean, m2, now_iso),
)
- self._conn.commit()
+ conn.commit()
def check(self, event_type: str, region: str, count: int) -> dict | None:
"""Check a value against the baseline.
@@ -86,7 +100,7 @@ class TemporalBaseline:
"""
key = self._key(event_type, region)
- row = self._conn.execute(
+ row = self._get_conn().execute(
"SELECT count, mean, m2 FROM baselines WHERE key = ?", (key,)
).fetchone()
diff --git a/src/world_intel_mcp/fetcher.py b/src/world_intel_mcp/fetcher.py
index 5b3afad..24f0d50 100644
--- a/src/world_intel_mcp/fetcher.py
+++ b/src/world_intel_mcp/fetcher.py
@@ -161,16 +161,14 @@ class Fetcher:
logger.debug("Circuit open for %s, trying stale cache", source)
return self._stale_fallback(effective_key, source)
- # Per-source rate limiting
- if yahoo_rate_limit:
- await self._yahoo_throttle()
- await self._source_throttle(source)
-
# Fetch with retries
client = await self._get_client()
last_error: Exception | None = None
for attempt in range(self.max_retries + 1):
+ if yahoo_rate_limit:
+ await self._yahoo_throttle()
+ await self._source_throttle(source)
try:
resp = await client.get(
url,
@@ -232,12 +230,11 @@ class Fetcher:
if not self.breaker.is_available(source):
return self._stale_fallback(effective_key, source)
- await self._source_throttle(source)
-
client = await self._get_client()
last_error: Exception | None = None
for attempt in range(self.max_retries + 1):
+ await self._source_throttle(source)
try:
resp = await client.get(
url,
@@ -288,12 +285,12 @@ class Fetcher:
if not self.breaker.is_available(source):
return BinaryResult(None, None, f"circuit open for {source}")
- await self._source_throttle(source)
client = await self._get_client()
last_error = "unknown error"
last_status: int | None = None
for attempt in range(self.max_retries + 1):
+ await self._source_throttle(source)
try:
resp = await client.get(
url, headers=headers, timeout=timeout or self.default_timeout
diff --git a/src/world_intel_mcp/runtime.py b/src/world_intel_mcp/runtime.py
index 2999c97..b829065 100644
--- a/src/world_intel_mcp/runtime.py
+++ b/src/world_intel_mcp/runtime.py
@@ -3,11 +3,8 @@
Extracted from server.py in the Phase 26 modularization so that the
domain tool modules under ``tools/`` and the server shell can share one
cache, one circuit breaker, one AOI store, and one fetcher without a
-circular import. Import-time side effects are deliberate and unchanged
-from the original server.py module body: constructing this module opens
-the cache database (honoring ``WORLD_INTEL_CACHE_DB``) exactly the way
-the server always has, which is what the registry tests' temp-cache
-override relies on.
+circular import. Infrastructure is initialized on first access so importing
+modules does not open databases.
"""
import logging
@@ -19,28 +16,40 @@ from .fetcher import Fetcher
logger = logging.getLogger("world-intel-mcp")
-cache = Cache()
-breaker = CircuitBreaker(failure_threshold=3, cooldown_seconds=300)
-# Same physical SQLite file the Cache above resolved to (fallback path
-# included); see analysis/aoi.py's module docstring for why AOIs live in
-# a dedicated table there rather than as Cache entries.
-_aoi_store = aoi.AOIStore(cache.db_path)
-aoi_store = _aoi_store
-
-# Vector store — optional, degrades gracefully if Qdrant unavailable.
-_vector_store = None
-try:
- from .vector_store import VectorStore, vector_dependencies_available
-
- if vector_dependencies_available():
- _vector_store = VectorStore(enabled=True)
- else:
- logger.info(
- "Vector store unavailable (qdrant_client / fastembed not installed)"
- )
-except Exception as exc:
- logger.info("Vector store unavailable: %s", exc)
-
-vector_store = _vector_store
-
-fetcher = Fetcher(cache=cache, breaker=breaker, vector_store=_vector_store)
+
+def __getattr__(name: str):
+ if name not in {"cache", "breaker", "aoi_store", "_aoi_store",
+ "vector_store", "_vector_store", "fetcher"}:
+ raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
+ _initialize()
+ return globals()[name]
+
+
+def _initialize() -> None:
+ global cache, breaker, aoi_store, _aoi_store, vector_store, _vector_store, fetcher
+
+ cache = Cache()
+ breaker = CircuitBreaker(failure_threshold=3, cooldown_seconds=300)
+ # Same physical SQLite file the Cache above resolved to (fallback path
+ # included); see analysis/aoi.py's module docstring for why AOIs live in
+ # a dedicated table there rather than as Cache entries.
+ _aoi_store = aoi.AOIStore(cache.db_path)
+ aoi_store = _aoi_store
+
+ # Vector store — optional, degrades gracefully if Qdrant unavailable.
+ _vector_store = None
+ try:
+ from .vector_store import VectorStore, vector_dependencies_available
+
+ if vector_dependencies_available():
+ _vector_store = VectorStore(enabled=True)
+ else:
+ logger.info(
+ "Vector store unavailable (qdrant_client / fastembed not installed)"
+ )
+ except Exception as exc:
+ logger.info("Vector store unavailable: %s", exc)
+
+ vector_store = _vector_store
+
+ fetcher = Fetcher(cache=cache, breaker=breaker, vector_store=_vector_store)
diff --git a/src/world_intel_mcp/server.py b/src/world_intel_mcp/server.py
index e620d38..c7a14ce 100644
--- a/src/world_intel_mcp/server.py
+++ b/src/world_intel_mcp/server.py
@@ -83,17 +83,19 @@ logger = logging.getLogger("world-intel-mcp")
server = Server("world-intel-mcp")
# Shared infrastructure lives in runtime.py (Phase 26 split); the local
-# aliases remain as the module's public face — external readers (and the
+# lazy aliases remain as the module's public face; external readers (and the
# registry tests) reach the live cache and vector store as server.cache,
# server._vector_store, etc.
from . import runtime as _runtime # noqa: E402
from . import tools as _tools_pkg # noqa: E402
-cache = _runtime.cache
-breaker = _runtime.breaker
-_aoi_store = _runtime.aoi_store
-_vector_store = _runtime.vector_store
-fetcher = _runtime.fetcher
+
+def __getattr__(name: str):
+ aliases = {"_aoi_store": "aoi_store", "_vector_store": "vector_store"}
+ if name in {"cache", "breaker", "fetcher", *aliases}:
+ return getattr(_runtime, aliases.get(name, name))
+ raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
+
# ---------------------------------------------------------------------------
# Tool registry — aggregated from the domain modules under tools/
@@ -142,8 +144,8 @@ async def call_tool(name: str, arguments: dict[str, Any] | None) -> list[TextCon
async def _run() -> None:
logger.info("World Intelligence MCP Server starting (%d tools)", len(TOOLS))
- if _vector_store:
- await _vector_store.start()
+ if _runtime.vector_store:
+ await _runtime.vector_store.start()
logger.info("Vector store worker started")
try:
async with stdio_server() as (read_stream, write_stream):
@@ -151,8 +153,8 @@ async def _run() -> None:
read_stream, write_stream, server.create_initialization_options()
)
finally:
- if _vector_store:
- await _vector_store.stop()
+ if _runtime.vector_store:
+ await _runtime.vector_store.stop()
def run() -> None:
diff --git a/src/world_intel_mcp/tests/test_cache_isolation.py b/src/world_intel_mcp/tests/test_cache_isolation.py
new file mode 100644
index 0000000..4e32a86
--- /dev/null
+++ b/src/world_intel_mcp/tests/test_cache_isolation.py
@@ -0,0 +1,117 @@
+"""Database imports and first-use paths must respect cache isolation."""
+
+import os
+import subprocess
+import sys
+from pathlib import Path
+
+import pytest
+
+from world_intel_mcp.analysis import spikes, temporal
+
+
+def test_all_module_imports_open_no_databases(tmp_path: Path) -> None:
+ home = tmp_path / "home"
+ xdg = tmp_path / "xdg"
+ home.mkdir()
+ xdg.mkdir()
+ env = dict(os.environ, HOME=str(home), XDG_CACHE_HOME=str(xdg),
+ WORLD_INTEL_CACHE_DB=str(xdg / "cache.db"))
+ script = """
+import importlib
+import sqlite3
+import sys
+from pathlib import Path
+
+connections = []
+def audit(event, args):
+ if event == 'sqlite3.connect':
+ connections.append(str(args[0]))
+sys.addaudithook(audit)
+
+import world_intel_mcp
+root = Path(world_intel_mcp.__file__).parent
+for path in sorted(root.rglob('*.py')):
+ relative = path.relative_to(root).with_suffix('')
+ if 'tests' in relative.parts:
+ continue
+ parts = relative.parts
+ if parts[-1] == '__init__':
+ parts = parts[:-1]
+ importlib.import_module('.'.join(('world_intel_mcp', *parts)))
+assert connections == [], connections
+"""
+ result = subprocess.run(
+ [sys.executable, "-c", script], env=env, capture_output=True, text=True,
+ timeout=30,
+ )
+ assert result.returncode == 0, result.stdout + result.stderr
+ assert list(home.rglob("*")) == []
+ assert list(xdg.rglob("*")) == []
+
+
[email protected]("module,cls,filename", [
+ (temporal, temporal.TemporalBaseline, "temporal.db"),
+ (spikes, spikes.KeywordSpikeDetector, "keyword_spikes.db"),
+])
[email protected]("location", ["xdg", "~/xdg", "", None])
+def test_baseline_path_resolved_on_first_use(
+ monkeypatch, tmp_path: Path, module, cls, filename: str, location: str | None,
+) -> None:
+ home = tmp_path / "home"
+ home.mkdir()
+ monkeypatch.setenv("HOME", str(home))
+ old_xdg = tmp_path / "before-first-use"
+ monkeypatch.setenv("XDG_CACHE_HOME", str(old_xdg))
+ monkeypatch.setenv("WORLD_INTEL_CACHE_DB", str(tmp_path / "main-cache.db"))
+ baseline = cls()
+ assert not old_xdg.exists()
+ assert list(home.iterdir()) == []
+
+ if location is None:
+ monkeypatch.delenv("XDG_CACHE_HOME")
+ else:
+ monkeypatch.setenv(
+ "XDG_CACHE_HOME", str(tmp_path / location) if location == "xdg" else location
+ )
+ expected_root = (
+ tmp_path / "xdg" if location == "xdg" else
+ home / "xdg" if location == "~/xdg" else home / ".cache"
+ )
+ try:
+ if module is temporal:
+ assert baseline.check("conflict", "UKR", 10) is None
+ baseline.record("conflict", "UKR", 10)
+ else:
+ assert baseline.detect_spikes({"conflict": 10}) == []
+ assert (expected_root / "world-intel-mcp" / filename).is_file()
+ assert not old_xdg.exists()
+ assert not (tmp_path / "main-cache.db").exists()
+ if location:
+ assert not (home / ".cache").exists()
+ finally:
+ if baseline._conn is not None:
+ baseline._conn.close()
+
+
[email protected]("module,cls", [
+ (temporal, temporal.TemporalBaseline),
+ (spikes, spikes.KeywordSpikeDetector),
+])
+def test_baseline_explicit_path_and_module_override(monkeypatch, tmp_path, module, cls):
+ explicit = tmp_path / "explicit.db"
+ override = tmp_path / "override.db"
+ baseline = cls(str(explicit))
+ default = cls()
+ monkeypatch.setattr(module, "_DB_PATH", str(override))
+ assert not explicit.exists()
+ assert not override.exists()
+ try:
+ baseline._get_conn()
+ default._get_conn()
+ assert explicit.is_file()
+ assert override.is_file()
+ finally:
+ for instance in (baseline, default):
+ if instance._conn is not None:
+ instance._conn.close()
diff --git a/src/world_intel_mcp/tests/test_fetcher_retry_spacing.py b/src/world_intel_mcp/tests/test_fetcher_retry_spacing.py
new file mode 100644
index 0000000..42bbbd2
--- /dev/null
+++ b/src/world_intel_mcp/tests/test_fetcher_retry_spacing.py
@@ -0,0 +1,44 @@
+"""Each retry must be spaced from the most recent attempt for its source."""
+
+import httpx
+import pytest
+
+from world_intel_mcp import fetcher as fetcher_mod
+from world_intel_mcp.fetcher import Fetcher
+
+
[email protected]("method", ["get_json", "get_text", "get_xml", "get_bytes"])
[email protected]("source", ["ripestat", "nominatim", "adsblol"])
+async def test_retries_respect_spacing_after_another_request(
+ cache, monkeypatch, method: str, source: str,
+) -> None:
+ now = 100.0
+ attempts = []
+ interleaved = False
+
+ def respond(request):
+ attempts.append(now)
+ return httpx.Response(503)
+
+ async with httpx.AsyncClient(transport=httpx.MockTransport(respond)) as client:
+ primary = Fetcher(cache, fetcher_mod.CircuitBreaker(), client=client)
+ other = Fetcher(cache, fetcher_mod.CircuitBreaker(), client=client, max_retries=0)
+
+ async def sleep(delay):
+ nonlocal now, interleaved
+ now += delay
+ if not interleaved:
+ # Another caller reaches this source while the first backs off.
+ interleaved = True
+ await getattr(other, method)("https://example.test/other", source)
+
+ monkeypatch.setattr(fetcher_mod.time, "time", lambda: now)
+ monkeypatch.setattr(fetcher_mod.asyncio, "sleep", sleep)
+ await getattr(primary, method)("https://example.test/primary", source)
+
+ assert len(attempts) == 4 # three primary attempts and one other caller
+ floor = fetcher_mod._SOURCE_RATE_LIMITS[source]
+ assert all(
+ later - earlier >= floor - 1e-9
+ for earlier, later in zip(attempts, attempts[1:])
+ ), attempts