Skip to content

160-tool MCP server for real-time global intelligence from free public APIs: markets, liquidity, prediction markets, SEC filings, conflict, military, aircraft-cohort early warning, migration and border flows, cyber, climate, news and more. Dated observation history, 29 research prompts, cited briefs, geofences, SSE dashboard. MIT, no paid API keys.

Liquidity: only the portal's own ECB 404 is a no-data answer

Phoenix Agent (mac-studio)df5fa7a66bad
diff --git a/CHANGELOG.md b/CHANGELOG.md
index a95abe5..5d436f7 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -53,10 +53,13 @@ responses say so. `limit=0` returns the full history.
`fetch_failure`, and name the failure in `data_gaps`: the HTTP status, no
response, an open circuit or an unexpected payload. Each stale copy served
counts in the circuit breaker's `total_stale_serves` (`intel_status`), and none
- is re-cached as fresh. Only parsed bodies enter the cache. ECB 404 responses
- are final no-data answers, without outage retries. An empty ECB response is
- one only when a start was requested; without one it is an unexpected payload
- and takes the outage path, as does JSON that is not an object.
+ is re-cached as fresh. Only parsed bodies enter the cache. An ECB 404 whose
+ body is empty or a JSON object (the portal's problem answer) is a final
+ no-data answer, without outage retries; any other 404, such as an HTML or
+ plain-text error page, is retried, counted as a breaker failure and takes the
+ outage path. An empty ECB response is one only when a start was requested;
+ without one it is an unexpected payload and takes the outage path, as does
+ JSON that is not an object.
Missing ECB roles and board rows are named; index bases, provisional status
and percentage-point change units are retained. ECB keys must specify every
dimension and request at most 50 series combinations.
diff --git a/src/world_intel_mcp/sources/liquidity.py b/src/world_intel_mcp/sources/liquidity.py
index fd24f27..f80a6ae 100644
--- a/src/world_intel_mcp/sources/liquidity.py
+++ b/src/world_intel_mcp/sources/liquidity.py
@@ -813,6 +813,24 @@ def parse_sdmx_json(data: dict, dataset: str) -> list[dict]:
_JSON_KINDS = {list: "array", str: "string", int: "number", float: "number", bool: "boolean", type(None): "null"}
+def _ecb_404_is_no_data(body: bytes) -> bool:
+ """Whether an ECB 404 is the portal's own answer to a key that matches
+ no series: an empty body or a JSON object (its problem+json, e.g.
+ "No Series was returned for the query"). A 404 with any other body
+ (an HTML or plain-text error page, JSON that is not an object) came
+ from something in front of the portal and takes the outage path.
+
+ Residual risk: a JSON-object 404 from a moved gateway would still
+ read as no-data.
+ """
+ if not body.strip():
+ return True
+ try:
+ return isinstance(json.loads(body), dict)
+ except (ValueError, RecursionError): # UnicodeError is a ValueError
+ return False
+
+
async def _ecb_fetch(
fetcher: Fetcher, dataset: str, key: str, last_n: int, start: date | None
) -> tuple[list[dict] | None, str | None]:
@@ -828,9 +846,9 @@ async def _ecb_fetch(
if fetched:
result = await fetcher.get_bytes(
f"{_ECB_DATA_URL.format(dataset=dataset, key=key)}?{urlencode(params)}",
- source=ECB_SOURCE, headers=_HEADERS, timeout=30.0,
+ source=ECB_SOURCE, headers=_HEADERS, timeout=30.0, accept_404=_ecb_404_is_no_data,
)
- if result.status == 404:
+ if result.status == 404 and result.error is None:
return None, f"ECB Data Portal: no series matches the key {dataset}/{key}"
# An unexpected payload takes the outage path (last good copy,
# marked) without a circuit-breaker failure: the transport worked.
diff --git a/src/world_intel_mcp/tests/test_liquidity_ecb_404.py b/src/world_intel_mcp/tests/test_liquidity_ecb_404.py
new file mode 100644
index 0000000..fc9599c
--- /dev/null
+++ b/src/world_intel_mcp/tests/test_liquidity_ecb_404.py
@@ -0,0 +1,176 @@
+"""ECB 404s: the portal's own "no series" answer versus an error page.
+
+The ECB Data Portal answers a key that matches no series with a 404
+whose body is empty or a JSON problem object; that stays a final
+no-data answer (not retried, not a breaker failure, no stale copy). A
+404 with any other body (an HTML or plain-text error page, JSON that is
+not an object) came from something in front of the portal: it is
+retried per the fetcher's rules, counted as a breaker failure, and the
+last good copy is served marked, with the failure named in data_gaps.
+
+Gaps / not covered: live ECB responses (fixtures and respx only). The
+residual risk stays: a JSON-object 404 from a moved gateway still reads
+as no-data.
+"""
+
+import time
+from pathlib import Path
+from unittest.mock import AsyncMock
+
+import httpx
+import pytest
+import respx
+
+from world_intel_mcp.cache import Cache
+from world_intel_mcp.circuit_breaker import CircuitBreaker
+from world_intel_mcp.fetcher import Fetcher
+from world_intel_mcp.sources import liquidity
+from world_intel_mcp.tests.test_sources_liquidity import (
+ _BSI, _ECB_404, _ECB_BSI_URL, _ECB_ILM_URL,
+ _fixed_clock_and_no_throttle, _mock_all,
+)
+
+_ECB_ROWS = {"euro_area_m1", "euro_area_m2", "euro_area_m3", "eurosystem_total_assets"}
+_BOARD_KEYS = (
+ f"liquidity:ecb:BSI:{liquidity._ECB_MONEY_KEY}:2:None",
+ f"liquidity:ecb:ILM:{liquidity._ECB_EUROSYSTEM_KEY}:2:None",
+)
+_NAMED = "upstream fetch failed (HTTP 404)"
+_HTML_404 = "<html><body><h1>404 Not Found</h1>nginx</body></html>"
+_ERROR_PAGES = [
+ pytest.param(httpx.Response(404, text=_HTML_404, headers={"content-type": "text/html"}), id="html"),
+ pytest.param(httpx.Response(404, text="Not Found", headers={"content-type": "text/plain"}), id="plain-text"),
+ pytest.param(httpx.Response(404, content=b"[]", headers={"content-type": "application/json"}), id="json-array"),
+]
+
+
+def _age(fetcher, seconds: float) -> None:
+ """Expire every cached copy and date it ``seconds`` ago."""
+ conn = fetcher.cache._get_conn()
+ conn.execute("UPDATE cache SET expires_at = ?, created_at = ?", (1, time.time() - seconds))
+ conn.commit()
+
+
+async def fetcher(tmp_path):
+ """Like test_sources_liquidity's fetcher (no retries), but closes its client and cache."""
+ f = Fetcher(cache=Cache(db_path=tmp_path / "liq.db"), breaker=CircuitBreaker(), default_timeout=5.0,
+ max_retries=0)
+ yield f
+ await f.close()
+ f.cache.close()
+
+
+def sleep(monkeypatch: pytest.MonkeyPatch) -> AsyncMock:
+ mock = AsyncMock()
+ monkeypatch.setattr("world_intel_mcp.fetcher.asyncio.sleep", mock)
+ return mock
+
+
[email protected]("response", _ERROR_PAGES)
+async def test_error_page_404_on_the_board_serves_the_marked_stale_copy(fetcher, sleep, response):
+ """The reviewer's probe N1: the board's fixed keys, a 2-hour-old good copy cached."""
+ _mock_all()
+ await liquidity.fetch_liquidity_board(fetcher)
+ _age(fetcher, 7200)
+ before = [fetcher.cache.get_stale_meta(k) for k in _BOARD_KEYS]
+ fetcher.max_retries = 2
+ bsi = respx.get(_ECB_BSI_URL).mock(return_value=response)
+ ilm = respx.get(_ECB_ILM_URL).mock(return_value=response)
+ sent = (bsi.call_count, ilm.call_count) # the routes also served the seeding call
+
+ board = await liquidity.fetch_liquidity_board(fetcher)
+
+ ecb = {r["indicator"]: r for r in board["indicators"] if r["source"] == liquidity.ECB_SOURCE}
+ assert set(ecb) == _ECB_ROWS
+ assert board["unavailable"] == []
+ for row in ecb.values():
+ assert row["served_from_stale_cache"] is True and row["freshness"]["stale"] is True
+ assert row["fetch_failure"] == _NAMED and row["freshness"]["fetch_failure"] == _NAMED
+ assert row["freshness"]["cache_age_seconds"] >= 7200
+ for name in _ECB_ROWS:
+ assert any(g.startswith(f"{name}: {_NAMED}; served from stale cache (") for g in board["data_gaps"]), name
+ assert not any("no series matches" in g for g in board["data_gaps"])
+ assert (bsi.call_count - sent[0], ilm.call_count - sent[1]) == (3, 3) # retried per the fetcher's rules
+ assert sleep.await_count == 4
+ state = fetcher.breaker.status()[liquidity.ECB_SOURCE]
+ assert state["total_failures"] == 2 and state["total_stale_serves"] == 2
+ assert [fetcher.cache.get_stale_meta(k) for k in _BOARD_KEYS] == before # never restamped
+
+
[email protected]("response", _ERROR_PAGES)
+async def test_error_page_404_is_named_cold_and_stale_on_a_single_series(fetcher, sleep, response):
+ key = liquidity._ECB_MONEY_KEY
+ route = respx.get(_ECB_BSI_URL).mock(return_value=response)
+ cold = await liquidity.fetch_ecb_series(fetcher, key=key)
+ assert cold["error"] == f"ECB Data Portal returned no data for BSI/{key}: {_NAMED}, no cached copy"
+ assert cold["data_gaps"] == [cold["error"]]
+ assert fetcher.breaker.status()[liquidity.ECB_SOURCE]["total_failures"] == 1
+
+ route.mock(return_value=httpx.Response(200, json=_BSI))
+ await liquidity.fetch_ecb_series(fetcher, key=key)
+ _age(fetcher, 7200)
+ route.mock(return_value=response)
+ warm = await liquidity.fetch_ecb_series(fetcher, key=key)
+ assert warm["series"] and all(s["served_from_stale_cache"] and s["fetch_failure"] == _NAMED
+ for s in warm["series"])
+ assert any(f": {_NAMED}; served from stale cache (" in g for g in warm["data_gaps"])
+ assert fetcher.breaker.status()[liquidity.ECB_SOURCE]["total_failures"] == 2
+
+
+async def test_the_portal_problem_404_stays_a_final_no_data_answer(fetcher, sleep):
+ """The live problem+json body ("No Series was returned for the query")."""
+ _mock_all()
+ await liquidity.fetch_liquidity_board(fetcher)
+ _age(fetcher, 7200)
+ fetcher.max_retries = 2
+ problem = httpx.Response(404, text=_ECB_404, headers={"content-type": "application/problem+json"})
+ bsi = respx.get(_ECB_BSI_URL).mock(return_value=problem)
+ ilm = respx.get(_ECB_ILM_URL).mock(return_value=problem)
+ sent = (bsi.call_count, ilm.call_count) # the routes also served the seeding call
+
+ board = await liquidity.fetch_liquidity_board(fetcher)
+
+ assert not any(r["source"] == liquidity.ECB_SOURCE for r in board["indicators"]) # no stale copy
+ errors = {u["indicator"]: u["error"] for u in board["unavailable"]}
+ assert set(errors) == _ECB_ROWS
+ assert errors["euro_area_m1"] == f"ECB Data Portal: no series matches the key BSI/{liquidity._ECB_MONEY_KEY}"
+ assert errors["eurosystem_total_assets"] == (
+ f"ECB Data Portal: no series matches the key ILM/{liquidity._ECB_EUROSYSTEM_KEY}"
+ )
+ assert (bsi.call_count - sent[0], ilm.call_count - sent[1]) == (1, 1)
+ sleep.assert_not_awaited()
+ state = fetcher.breaker.status()[liquidity.ECB_SOURCE]
+ assert state["total_failures"] == 0 and state["total_stale_serves"] == 0
+
+
[email protected]("body, no_data", [
+ pytest.param(b"", True, id="empty"),
+ pytest.param(b" \n", True, id="whitespace"),
+ pytest.param(_ECB_404.encode(), True, id="live-problem-json"),
+ pytest.param(b'{"title":"No results"}', True, id="json-object"),
+ pytest.param(_HTML_404.encode(), False, id="html"),
+ pytest.param(b"Not Found", False, id="plain-text"),
+ pytest.param(b"[]", False, id="json-array"),
+ pytest.param(b"null", False, id="json-null"),
+ pytest.param(b'"No results"', False, id="json-string"),
+ pytest.param(b"{broken", False, id="broken-json"),
+ pytest.param(b"\x80{}", False, id="not-utf8"),
+ pytest.param(b"[" * 100_000, False, id="nesting-past-recursion-limit"),
+])
+def test_ecb_404_body_classification(body, no_data):
+ assert liquidity._ecb_404_is_no_data(body) is no_data
+
+
+def test_changelog_states_the_ecb_404_rule():
+ text = " ".join((Path(__file__).parents[3] / "CHANGELOG.md").read_text().split())
+ assert (
+ "An ECB 404 whose body is empty or a JSON object (the portal's problem answer) is a final "
+ "no-data answer, without outage retries; any other 404, such as an HTML or plain-text error "
+ "page, is retried, counted as a breaker failure and takes the outage path."
+ ) in text
+ assert "ECB 404 responses are final no-data answers" not in text