Skip to content

160-tool MCP server for real-time global intelligence from free public APIs: markets, liquidity, prediction markets, SEC filings, conflict, military, aircraft-cohort early warning, migration and border flows, cyber, climate, news and more. Dated observation history, 29 research prompts, cited briefs, geofences, SSE dashboard. MIT, no paid API keys.

Weekly trends: name failed or history-less inputs instead of zero

Phoenix Agent (mac-studio)2d6d777cbefa
diff --git a/src/world_intel_mcp/analysis/alerts.py b/src/world_intel_mcp/analysis/alerts.py
index d61007a..9168c74 100644
--- a/src/world_intel_mcp/analysis/alerts.py
+++ b/src/world_intel_mcp/analysis/alerts.py
@@ -8,8 +8,10 @@ to avoid circular dependencies with source modules.
import asyncio
import logging
import math
+import os
import sqlite3
from datetime import datetime, timezone
+from pathlib import Path
logger = logging.getLogger("world-intel-mcp.analysis.alerts")
@@ -234,6 +236,32 @@ async def fetch_alert_digest(fetcher) -> dict:
# Weekly Trends
# ---------------------------------------------------------------------------
+# A metric needs this many observations before its volatility is reported.
+_MIN_TREND_OBSERVATIONS = 5
+
+
+def _read_baselines(db_path: str) -> tuple[list[tuple], str | None]:
+ """Every baseline row, or the reason none could be read.
+
+ The file is opened read-only: reading trends must never leave an empty
+ database where the baselines were expected.
+ """
+ if not os.path.exists(db_path):
+ return [], f"no temporal baseline database at {db_path}"
+ try:
+ conn = sqlite3.connect(f"{Path(db_path).absolute().as_uri()}?mode=ro", uri=True)
+ try:
+ rows = conn.execute(
+ "SELECT key, count, mean, m2, updated_at FROM baselines"
+ ).fetchall()
+ finally:
+ conn.close()
+ except (sqlite3.Error, OSError) as exc:
+ logger.warning("Failed to read temporal baselines: %s", exc)
+ return [], f"could not read {db_path}: {exc}"
+ return rows, None
+
+
async def fetch_weekly_trends(fetcher) -> dict:
"""Analyze weekly trends from temporal baselines.
@@ -241,33 +269,63 @@ async def fetch_weekly_trends(fetcher) -> dict:
(coefficient of variation) for each tracked metric, and calls
temporal_anomalies to get current deviations.
+ A failed input never reads as zero. ``data_gaps`` names every input
+ that failed or has no history to judge by. When the baseline database
+ is missing or unreadable, ``trend_count`` is None and ``error`` says
+ why; when anomaly detection fails, ``current_anomaly_count`` is None.
+
Args:
fetcher: Shared HTTP fetcher with caching and circuit breaking.
Returns:
- Dict with trends, trend_count, analysis_period, source.
+ Dict with trends, trend_count, current_anomalies,
+ current_anomaly_count, data_gaps, analysis_period, source, and
+ error when the baselines could not be read.
"""
from ..analysis.temporal import _default_db_path
from ..sources import intelligence
- now = datetime.now(timezone.utc)
+ db_path = _default_db_path()
+ # Anomaly detection records this cycle's observations and creates the
+ # database on first use, so look first: a baseline born in this call has
+ # no history, which is not the same as a quiet week.
+ baseline_existed = os.path.exists(db_path)
+ data_gaps: list[str] = []
# Get current anomalies
- anomaly_data = await _safe_fetch(
- intelligence.fetch_temporal_anomalies(fetcher), "temporal_anomalies"
- )
+ current_anomalies: list[dict] = []
+ current_anomaly_count: int | None = None
+ try:
+ anomaly_data = await intelligence.fetch_temporal_anomalies(fetcher)
+ except Exception as exc:
+ logger.warning("Weekly trends: temporal anomalies failed: %s", exc)
+ data_gaps.append(f"Current anomalies: {type(exc).__name__}: {exc}")
+ else:
+ anomalies = (
+ anomaly_data.get("anomalies") if isinstance(anomaly_data, dict) else None
+ )
+ if isinstance(anomalies, list):
+ current_anomalies = anomalies
+ current_anomaly_count = len(anomalies)
+ data_gaps.extend(
+ f"Current anomalies: {gap}" for gap in anomaly_data.get("data_gaps") or []
+ )
+ else:
+ data_gaps.append(
+ "Current anomalies: anomaly detection returned no anomaly list"
+ )
# Read baselines from SQLite
+ rows, baseline_problem = _read_baselines(db_path)
trends: list[dict] = []
+ trend_count: int | None = None
- try:
- conn = sqlite3.connect(_default_db_path())
- rows = conn.execute(
- "SELECT key, count, mean, m2, updated_at FROM baselines WHERE count >= 5"
- ).fetchall()
- conn.close()
-
+ if baseline_problem is not None:
+ data_gaps.append(f"Baselines: {baseline_problem}")
+ else:
for key, n, mean, m2, updated_at in rows:
+ if n < _MIN_TREND_OBSERVATIONS:
+ continue
parts = key.split(":")
if len(parts) < 2:
continue
@@ -293,22 +351,33 @@ async def fetch_weekly_trends(fetcher) -> dict:
"volatility_cv": round(cv, 1),
"last_updated": updated_at,
})
-
- except (sqlite3.Error, OSError) as exc:
- logger.warning("Failed to read temporal baselines: %s", exc)
+ trend_count = len(trends)
+
+ if not baseline_existed:
+ data_gaps.append(
+ f"Baselines: none existed at {db_path} before this call, so "
+ "trends and anomalies have no history yet"
+ )
+ elif not any(row[1] >= _MIN_TREND_OBSERVATIONS for row in rows):
+ data_gaps.append(
+ "Baselines: no tracked metric has the "
+ f"{_MIN_TREND_OBSERVATIONS} observations volatility needs yet "
+ f"({len(rows)} tracked)"
+ )
# Sort by volatility descending
trends.sort(key=lambda t: t.get("volatility_cv", 0), reverse=True)
- # Attach current anomalies
- current_anomalies = anomaly_data.get("anomalies", [])
-
- return {
+ result = {
"trends": trends[:50],
- "trend_count": len(trends),
+ "trend_count": trend_count,
"current_anomalies": current_anomalies,
- "current_anomaly_count": len(current_anomalies),
+ "current_anomaly_count": current_anomaly_count,
+ "data_gaps": data_gaps,
"analysis_period": "weekly (by weekday+month seasonality)",
"source": "temporal-weekly-trends",
"timestamp": _utc_now_iso(),
}
+ if baseline_problem is not None:
+ result["error"] = f"Temporal baselines unavailable: {baseline_problem}"
+ return result
diff --git a/src/world_intel_mcp/tests/test_weekly_trends_gaps.py b/src/world_intel_mcp/tests/test_weekly_trends_gaps.py
new file mode 100644
index 0000000..1b83e3e
--- /dev/null
+++ b/src/world_intel_mcp/tests/test_weekly_trends_gaps.py
@@ -0,0 +1,254 @@
+"""intel_weekly_trends never reads a failed or history-less input as zero.
+
+The baseline path always points under tmp_path. Most tests mock anomaly
+detection at the source-function boundary; the two first-use routes run
+the real detector (a fresh TemporalBaseline against a mocked live posture
+feed), so the database it creates or fails to create is the one trends
+then read.
+"""
+
+import sqlite3
+from pathlib import Path
+
+import pytest
+
+from world_intel_mcp.analysis import temporal
+from world_intel_mcp.analysis.alerts import fetch_weekly_trends
+from world_intel_mcp.sources import intelligence, military
+
+_TS = "2026-10-12T00:00:00Z"
+
+
+def _returns(payload):
+ async def _fake(*args, **kwargs):
+ return payload
+
+ return _fake
+
+
+def _raises(exc: BaseException):
+ async def _fake(*args, **kwargs):
+ raise exc
+
+ return _fake
+
+
+def _seed(db: Path, rows: list[tuple]) -> None:
+ db.parent.mkdir(parents=True, exist_ok=True)
+ conn = sqlite3.connect(db)
+ conn.execute(
+ """CREATE TABLE baselines (
+ key TEXT PRIMARY KEY,
+ count INTEGER NOT NULL DEFAULT 0,
+ mean REAL NOT NULL DEFAULT 0.0,
+ m2 REAL NOT NULL DEFAULT 0.0,
+ updated_at TEXT NOT NULL
+ )"""
+ )
+ conn.executemany("INSERT INTO baselines VALUES (?, ?, ?, ?, ?)", rows)
+ conn.commit()
+ conn.close()
+
+
+def live_detector(monkeypatch):
+ """The real anomaly detector on a fresh baseline whose path resolves on
+ first use, fed by a live-looking posture feed and ACLED reply."""
+ baseline = temporal.TemporalBaseline()
+ monkeypatch.setattr(temporal, "_DB_PATH", None)
+ monkeypatch.setattr(intelligence, "_temporal", baseline)
+ monkeypatch.setattr(
+ military,
+ "fetch_theater_posture",
+ _returns({"theaters": {"european": {"count": 12}}}),
+ )
+ monkeypatch.setattr(
+ intelligence, "acled_query", _returns({"data": [{"country": "Ukraine"}]})
+ )
+ try:
+ yield baseline
+ finally:
+ if baseline._conn is not None:
+ baseline._conn.close()
+
+
+async def test_missing_baselines_and_failed_anomalies_are_named_not_zero(
+ fetcher, tmp_path: Path, monkeypatch,
+) -> None:
+ db = tmp_path / "temporal.db"
+ monkeypatch.setattr(temporal, "_DB_PATH", str(db))
+ monkeypatch.setattr(
+ intelligence,
+ "fetch_temporal_anomalies",
+ _raises(PermissionError(13, "Permission denied")),
+ )
+
+ result = await fetch_weekly_trends(fetcher)
+
+ assert result["trend_count"] is None
+ assert result["current_anomaly_count"] is None
+ assert result["trends"] == []
+ assert result["current_anomalies"] == []
+ assert result["error"] == (
+ f"Temporal baselines unavailable: no temporal baseline database at {db}"
+ )
+ assert result["data_gaps"] == [
+ "Current anomalies: PermissionError: [Errno 13] Permission denied",
+ f"Baselines: no temporal baseline database at {db}",
+ ]
+ assert not db.exists(), "reading trends created a database"
+
+
[email protected]("content", ["not a database", "no baselines table"])
+async def test_unreadable_baselines_are_an_error_and_anomalies_survive(
+ fetcher, tmp_path: Path, monkeypatch, content: str,
+) -> None:
+ db = tmp_path / "temporal.db"
+ if content == "not a database":
+ db.write_bytes(b"this is not an SQLite database file. " * 40)
+ reason = "file is not a database"
+ else:
+ sqlite3.connect(db).close()
+ reason = "no such table: baselines"
+ monkeypatch.setattr(temporal, "_DB_PATH", str(db))
+ monkeypatch.setattr(
+ intelligence,
+ "fetch_temporal_anomalies",
+ _returns({"anomalies": [{"event_type": "conflict", "z_score": 2.4}], "data_gaps": []}),
+ )
+
+ result = await fetch_weekly_trends(fetcher)
+
+ assert result["trend_count"] is None
+ assert result["current_anomaly_count"] == 1
+ assert result["data_gaps"] == [f"Baselines: could not read {db}: {reason}"]
+ assert result["error"] == f"Temporal baselines unavailable: could not read {db}: {reason}"
+
+
+async def test_anomaly_reply_without_a_list_is_a_gap_not_zero(
+ fetcher, tmp_path: Path, monkeypatch,
+) -> None:
+ db = tmp_path / "temporal.db"
+ _seed(db, [("conflict:UKR:Monday:January", 10, 20.0, 900.0, _TS)])
+ monkeypatch.setattr(temporal, "_DB_PATH", str(db))
+ monkeypatch.setattr(intelligence, "fetch_temporal_anomalies", _returns({}))
+
+ result = await fetch_weekly_trends(fetcher)
+
+ assert result["trend_count"] == 1
+ assert result["current_anomaly_count"] is None
+ assert result["current_anomalies"] == []
+ assert result["data_gaps"] == [
+ "Current anomalies: anomaly detection returned no anomaly list"
+ ]
+ assert "error" not in result
+
+
+async def test_anomaly_detections_own_gaps_are_carried(
+ fetcher, tmp_path: Path, monkeypatch,
+) -> None:
+ db = tmp_path / "temporal.db"
+ _seed(db, [("conflict:UKR:Monday:January", 10, 20.0, 900.0, _TS)])
+ monkeypatch.setattr(temporal, "_DB_PATH", str(db))
+ monkeypatch.setattr(
+ intelligence,
+ "fetch_temporal_anomalies",
+ _returns({"anomalies": [], "data_gaps": ["acled_fetch_failed"]}),
+ )
+
+ result = await fetch_weekly_trends(fetcher)
+
+ assert result["current_anomaly_count"] == 0
+ assert result["data_gaps"] == ["Current anomalies: acled_fetch_failed"]
+
+
+async def test_healthy_inputs_report_no_gaps(
+ fetcher, tmp_path: Path, monkeypatch,
+) -> None:
+ db = tmp_path / "temporal.db"
+ _seed(db, [("conflict:UKR:Monday:January", 10, 20.0, 900.0, _TS)])
+ monkeypatch.setattr(temporal, "_DB_PATH", str(db))
+ monkeypatch.setattr(
+ intelligence, "fetch_temporal_anomalies", _returns({"anomalies": [], "data_gaps": []})
+ )
+
+ result = await fetch_weekly_trends(fetcher)
+
+ assert result["trend_count"] == 1
+ assert result["current_anomaly_count"] == 0
+ assert result["data_gaps"] == []
+ assert "error" not in result
+
+
+async def test_young_baseline_says_it_lacks_history(
+ fetcher, tmp_path: Path, monkeypatch,
+) -> None:
+ db = tmp_path / "temporal.db"
+ _seed(
+ db,
+ [
+ ("conflict:UKR:Monday:January", 4, 20.0, 30.0, _TS),
+ ("outage:EGY", 1, 5.0, 0.0, _TS),
+ ],
+ )
+ monkeypatch.setattr(temporal, "_DB_PATH", str(db))
+ monkeypatch.setattr(
+ intelligence, "fetch_temporal_anomalies", _returns({"anomalies": [], "data_gaps": []})
+ )
+
+ result = await fetch_weekly_trends(fetcher)
+
+ assert result["trend_count"] == 0
+ assert result["data_gaps"] == [
+ "Baselines: no tracked metric has the 5 observations volatility needs yet (2 tracked)"
+ ]
+ assert "error" not in result
+
+
+async def test_baseline_born_in_this_call_is_flagged_and_old_one_left_alone(
+ fetcher, tmp_path: Path, monkeypatch, live_detector,
+) -> None:
+ """The upgrade route: XDG_CACHE_HOME is set while the history still sits
+ in ~/.cache, which is never moved."""
+ home = tmp_path / "home"
+ legacy = home / ".cache" / "world-intel-mcp" / "temporal.db"
+ _seed(legacy, [("military_flights:european:Monday:October", 40, 11.0, 390.0, _TS)])
+ xdg = tmp_path / "xdg"
+ monkeypatch.setenv("HOME", str(home))
+ monkeypatch.setenv("XDG_CACHE_HOME", str(xdg))
+ created = xdg / "world-intel-mcp" / "temporal.db"
+
+ result = await fetch_weekly_trends(fetcher)
+
+ assert created.is_file()
+ assert result["trend_count"] == 0
+ assert result["current_anomaly_count"] == 0
+ assert result["data_gaps"] == [
+ f"Baselines: none existed at {created} before this call, so trends "
+ "and anomalies have no history yet"
+ ]
+ rows = sqlite3.connect(legacy).execute("SELECT key, count FROM baselines").fetchall()
+ assert rows == [("military_flights:european:Monday:October", 40)]
+
+
+async def test_baseline_location_that_cannot_be_created_is_named(
+ fetcher, tmp_path: Path, monkeypatch, live_detector,
+) -> None:
+ """The unwritable route: the baseline directory cannot be made (here
+ because the cache home is a regular file), so detection fails and no
+ database exists to read."""
+ cache_home = tmp_path / "cache-home-is-a-file"
+ cache_home.write_text("")
+ monkeypatch.setenv("XDG_CACHE_HOME", str(cache_home))
+ db = cache_home / "world-intel-mcp" / "temporal.db"
+
+ result = await fetch_weekly_trends(fetcher)
+
+ assert result["trend_count"] is None
+ assert result["current_anomaly_count"] is None
+ assert result["error"] == (
+ f"Temporal baselines unavailable: no temporal baseline database at {db}"
+ )
+ anomaly_gap, baseline_gap = result["data_gaps"]
+ assert anomaly_gap.startswith("Current anomalies: NotADirectoryError: ")
+ assert baseline_gap == f"Baselines: no temporal baseline database at {db}"