Skip to content

160-tool MCP server for real-time global intelligence from free public APIs: markets, liquidity, prediction markets, SEC filings, conflict, military, aircraft-cohort early warning, migration and border flows, cyber, climate, news and more. Dated observation history, 29 research prompts, cited briefs, geofences, SSE dashboard. MIT, no paid API keys.

UCDP: a page that could not be fetched is a named gap

Phoenix Agent (mac-studio)9bb82eba2b7b
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 00db472..cb637cb 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -287,6 +287,12 @@ source here, and no proxy is substituted for any of these measures.
country and days only, so a short list cached for a small `limit` came
back, for 15 minutes, to a call that asked for more rows, and a count
below the limit then read as complete. `limit` is now part of both keys.
+- `intel_ucdp_events` dropped a page that failed after the first one (an
+ outage, or the circuit breaker skipping it) and returned the shorter
+ list with no marker. Such a result now carries `degraded`, reason
+ `ucdp_partial`, `pages_failed`, `pages_total` and a `data_gaps` entry
+ saying count and fatalities are lower bounds, and the conflict-trend
+ prompt tells the model to report the list as incomplete.
### Added: tropical cyclone forecast tracks
- `intel_cyclones` and `intel_jtwc_cyclones` take `include_forecast`
diff --git a/src/world_intel_mcp/guide_prompts.py b/src/world_intel_mcp/guide_prompts.py
index 68baaa2..111cc7f 100644
--- a/src/world_intel_mcp/guide_prompts.py
+++ b/src/world_intel_mcp/guide_prompts.py
@@ -338,7 +338,10 @@ PROMPTS: tuple[ResearchPrompt, ...] = (
"has no country filter, so keep the events in {country} "
"yourself, split them at {days} days ago by date_start, and say "
"that you filtered. limit is the page size and every page is "
- "fetched, so it does not cap the result",
+ "requested, so it does not cap the result. When a page could "
+ "not be fetched the result carries reason ucdp_partial and "
+ "pages_failed: its count and fatalities are then lower bounds, "
+ "so say the list is incomplete",
args=(("days", "{span}"), ("limit", 1000)),
),
Step(
diff --git a/src/world_intel_mcp/sources/conflict.py b/src/world_intel_mcp/sources/conflict.py
index d618067..c66708f 100644
--- a/src/world_intel_mcp/sources/conflict.py
+++ b/src/world_intel_mcp/sources/conflict.py
@@ -272,8 +272,9 @@ async def fetch_ucdp_events(
fetcher: Shared HTTP fetcher with caching and circuit breaking.
days: Only include events whose ``date_start`` is within this
many days from now.
- limit: Page size (max 1000 per page). Every page is fetched, so
- this does not cap the number of events returned.
+ limit: Page size (max 1000 per page). Every page is requested,
+ so this does not cap the number of events returned; a page
+ that cannot be fetched is named in the result (below).
Returns:
Dict with events list, count, total fatalities, query params,
@@ -282,7 +283,11 @@ async def fetch_ucdp_events(
``degraded`` and ``reason`` (``ucdp_unconfigured`` or
``ucdp_fetch_failed``) next to the empty events list, the same
shape as ACLED's fetch failure, so a failure never reads as zero
- events.
+ events. When a later page cannot be fetched (an outage, or the
+ circuit breaker skipping it), the events from the other pages are
+ kept and the result carries ``degraded``, ``reason``
+ ``ucdp_partial``, ``pages_failed``, ``pages_total`` and a
+ ``data_gaps`` entry: count and fatalities are then lower bounds.
"""
now = datetime.now(timezone.utc)
cutoff = now - timedelta(days=days)
@@ -338,6 +343,7 @@ async def fetch_ucdp_events(
all_results = list(page1_data.get("Result", []))
# Fetch remaining pages in parallel
+ pages_failed: list[int] = []
if total_pages > 1:
page_tasks = [
fetcher.get_json(
@@ -352,9 +358,11 @@ async def fetch_ucdp_events(
for p in range(1, total_pages)
]
page_results = await asyncio.gather(*page_tasks)
- for page_data in page_results:
- if page_data is not None:
+ for page_number, page_data in enumerate(page_results, start=1):
+ if isinstance(page_data, dict):
all_results.extend(page_data.get("Result", []))
+ else:
+ pages_failed.append(page_number)
# Filter to recent events and extract fields
events = []
@@ -434,7 +442,7 @@ async def fetch_ucdp_events(
}
)
- return {
+ result = {
"events": events,
"count": len(events),
"total_fatalities_best": total_fatalities_best,
@@ -442,6 +450,20 @@ async def fetch_ucdp_events(
"source": "ucdp",
"timestamp": now.strftime("%Y-%m-%dT%H:%M:%SZ"),
}
+ if pages_failed:
+ # A dropped page would otherwise shrink the count with no trace.
+ logger.warning(
+ "UCDP: %d of %d pages could not be fetched", len(pages_failed), total_pages
+ )
+ result["degraded"] = True
+ result["reason"] = "ucdp_partial"
+ result["pages_failed"] = pages_failed
+ result["pages_total"] = total_pages
+ result["data_gaps"] = [
+ f"UCDP: {len(pages_failed)} of {total_pages} pages could not be "
+ "fetched, so count and total_fatalities_best are lower bounds"
+ ]
+ return result
# ---------------------------------------------------------------------------
diff --git a/src/world_intel_mcp/tests/test_sources_ucdp.py b/src/world_intel_mcp/tests/test_sources_ucdp.py
index b50ed6a..aa2a406 100644
--- a/src/world_intel_mcp/tests/test_sources_ucdp.py
+++ b/src/world_intel_mcp/tests/test_sources_ucdp.py
@@ -116,3 +116,64 @@ async def test_a_different_page_size_does_not_reuse_cached_pages(cache, monkeypa
assert sorted(e["id"] for e in first["events"]) == list(range(7))
assert sorted(e["id"] for e in second["events"]) == list(range(7))
assert upstream.pages(seen) == [("3", "0"), ("3", "1"), ("3", "2")]
+
+
+class _PageDown(_Upstream):
+ """Like _Upstream, but the listed pages answer 503."""
+
+ def __init__(self, events: list[int], down: set[int]):
+ super().__init__(events=events)
+ self.down = down
+
+ def __call__(self, request: httpx.Request) -> httpx.Response:
+ if int(request.url.params["page"]) in self.down:
+ self.requests.append(request)
+ return httpx.Response(503)
+ return super().__call__(request)
+
+
+async def test_a_failed_later_page_is_a_named_gap_not_a_silent_drop(cache, monkeypatch) -> None:
+ """Pages of 3 over 7 events with page 1 down: the 4 events from pages 0
+ and 2 stay, and the result says the count is a lower bound."""
+ monkeypatch.setenv("UCDP_ACCESS_TOKEN", "t")
+ upstream = _PageDown(events=list(range(7)), down={1})
+ result = await _fetch(cache, upstream, days=30, limit=3)
+ assert sorted(e["id"] for e in result["events"]) == [0, 1, 2, 6]
+ assert (result["count"], result["total_fatalities_best"]) == (4, 4)
+ assert (result["degraded"], result["reason"]) == (True, "ucdp_partial")
+ assert (result["pages_failed"], result["pages_total"]) == ([1], 3)
+ assert result["data_gaps"] == [
+ "UCDP: 1 of 3 pages could not be fetched, so count and "
+ "total_fatalities_best are lower bounds"
+ ]
+ assert "error" not in result
+
+
+async def test_a_page_the_breaker_skips_is_also_named(cache, monkeypatch) -> None:
+ """Once the breaker opens on a failed page, a later call skips that page
+ without a request; it must still be named, not dropped."""
+ monkeypatch.setenv("UCDP_ACCESS_TOKEN", "t")
+ upstream = _PageDown(events=list(range(7)), down={2})
+ async with httpx.AsyncClient(transport=httpx.MockTransport(upstream)) as client:
+ fetcher = Fetcher(
+ cache, CircuitBreaker(failure_threshold=1), max_retries=0, client=client
+ )
+ first = await fetch_ucdp_events(fetcher, days=30, limit=3)
+ upstream.down = set()
+ seen = len(upstream.requests)
+ second = await fetch_ucdp_events(fetcher, days=30, limit=3)
+ assert first["pages_failed"] == [2]
+ assert len(upstream.requests) == seen
+ assert (second["count"], second["pages_failed"], second["reason"]) == (6, [2], "ucdp_partial")
+
+
+async def test_all_pages_fetched_carries_no_partial_marker(cache, monkeypatch) -> None:
+ monkeypatch.setenv("UCDP_ACCESS_TOKEN", "t")
+ upstream = _Upstream(events=list(range(7)))
+ result = await _fetch(cache, upstream, days=30, limit=3)
+ assert result["count"] == 7
+ for key in ("degraded", "reason", "pages_failed", "pages_total", "data_gaps"):
+ assert key not in result