Skip to content

160-tool MCP server for real-time global intelligence from free public APIs: markets, liquidity, prediction markets, SEC filings, conflict, military, aircraft-cohort early warning, migration and border flows, cyber, climate, news and more. Dated observation history, 29 research prompts, cited briefs, geofences, SSE dashboard. MIT, no paid API keys.

History: quote units say what the number is; a raising extractor degrades

Phoenix Agent (mac-studio)daeb0826c255
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 9f7af3e..4b495db 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -21,7 +21,8 @@
`_stale` marker. A replay's observations are counted but not stored: a
stale replay is not a new observation. A live capture that is degraded
for another reason (empty, partial, marked degraded) still stores the
- points it has.
+ points it has. A capture whose extractor raised is degraded too, with
+ the error as its reason; its payload is kept.
- Payload bodies are kept once per distinct payload as compressed JSON
(top-level fetch-time stamps and the stale markers are ignored when
hashing, so a fresh and a stale copy of the same data read unchanged).
@@ -45,7 +46,10 @@
payload names a release time) and `collected_at`. Built-in extractors:
NOAA Kp, USGS per-event magnitude, FRED-sourced central bank rates
(curated rows are skipped), Yahoo index and commodity quotes (null
- `observed_at`: the payload carries no quote time).
+ `observed_at`: the payload carries no quote time). An index level's unit
+ is `index points`, with the quote currency in its label (`^GSPC (USD)`);
+ a commodity future's unit is its exchange quotation unit (`USD per troy
+ ounce`, `US cents per bushel`).
`history.register_extractor()` is the public API for new sources; its
`expected=N` declares how many points a complete payload yields. The
Yahoo quote sources declare one per default symbol, so a run where some
diff --git a/src/world_intel_mcp/history.py b/src/world_intel_mcp/history.py
index 4041392..509158c 100644
--- a/src/world_intel_mcp/history.py
+++ b/src/world_intel_mcp/history.py
@@ -71,8 +71,8 @@ only when the publisher documents UTC); a point with an unparseable date
or a non-finite value is skipped and counted in the capture's
``extract_error``. Register at import time of your source module: the
collector imports a source's module before it records that source's
-capture. A registered extractor that finds nothing in a non-error payload
-marks the capture ``degraded``; pass ``expected=N`` when a
+capture. A registered extractor that finds nothing in a non-error payload,
+or raises, marks the capture ``degraded``; pass ``expected=N`` when a
complete payload yields N points (one per requested symbol), and a
capture with fewer is ``degraded`` as partial.
@@ -365,7 +365,56 @@ def _extract_central_bank_rates(payload: dict) -> list[Observation]:
return points
-def _quote_extractor(list_key: str, label_key: str, unit_key: str | None):
+# Exchange quotation units of the commodity futures the collector reads
+# (the commodity payload carries no currency). The currency half was
+# checked against Yahoo's chart meta on 2026-10-17 (USD for COMEX and
+# NYMEX, USX = US cents for the CBOT grains); the per-unit half is each
+# contract's quotation unit.
+COMMODITY_UNITS = {
+ "GC=F": "USD per troy ounce",
+ "SI=F": "USD per troy ounce",
+ "CL=F": "USD per barrel",
+ "BZ=F": "USD per barrel",
+ "NG=F": "USD per MMBtu",
+ "ZC=F": "US cents per bushel",
+ "ZW=F": "US cents per bushel",
+ "ZS=F": "US cents per bushel",
+}
+# Yahoo reports an index level with its constituents' currency; the level
+# itself is in index points, so the currency goes in the label.
+_INDEX_UNIT = "index points"
+
+
+def _market_quote_unit(quote: dict) -> str | None:
+ symbol = str(quote.get("symbol") or "")
+ if symbol in _DEFAULT_INDEX_SYMBOLS:
+ return _INDEX_UNIT
+ if symbol.startswith("^"):
+ return None # a Yahoo index or rate whose unit this table does not know
+ return quote.get("currency")
+
+
+def _market_quote_label(quote: dict) -> str:
+ symbol = str(quote.get("symbol") or "")
+ currency = quote.get("currency")
+ if symbol in _DEFAULT_INDEX_SYMBOLS and isinstance(currency, str) and currency:
+ return f"{symbol} ({currency})"
+ return symbol
+
+
+def _commodity_unit(quote: dict) -> str | None:
+ return COMMODITY_UNITS.get(str(quote.get("symbol") or ""))
+
+
+def _commodity_label(quote: dict) -> str:
+ return quote.get("name") or str(quote.get("symbol") or "")
+
+
+def _quote_extractor(
+ list_key: str,
+ unit_of: Callable[[dict], str | None],
+ label_of: Callable[[dict], str],
+):
def _extract(payload: dict) -> list[Observation]:
"""Yahoo quotes. The source module keeps price and change only, not
the quote time, so observed_at is NULL (unknown), not collected_at."""
@@ -377,8 +426,7 @@ def _quote_extractor(list_key: str, label_key: str, unit_key: str | None):
continue
points.append(
Observation(f"{symbol}:price", price, observed_at=None,
- unit=quote.get(unit_key) if unit_key else None,
- label=quote.get(label_key) or symbol)
+ unit=unit_of(quote), label=label_of(quote) or symbol)
)
return points
@@ -387,11 +435,11 @@ def _quote_extractor(list_key: str, label_key: str, unit_key: str | None):
# The collector runs both with their defaults: one point per symbol.
register_extractor(
- "market_quotes", _quote_extractor("quotes", "symbol", "currency"),
+ "market_quotes", _quote_extractor("quotes", _market_quote_unit, _market_quote_label),
expected=len(_DEFAULT_INDEX_SYMBOLS),
)
register_extractor(
- "commodity_quotes", _quote_extractor("commodities", "name", None),
+ "commodity_quotes", _quote_extractor("commodities", _commodity_unit, _commodity_label),
expected=len(_COMMODITY_SYMBOLS),
)
@@ -504,7 +552,8 @@ def classify(outcome: SourceOutcome) -> _Classified:
``error`` when the run produced no usable payload. ``degraded`` when it
was a stale-cache replay, or live but marked degraded, empty, partial,
- or without the measures its extractor expects. Otherwise ``ok``. A
+ without the measures its extractor expects, or its extractor raised
+ (the payload is kept; the error is the reason). Otherwise ``ok``. A
stale-cache replay keeps its extracted count but no observations. The
reason holds no per-run numbers for a replay (its copy age and fetch
count are fields), so successive replays share one reason.
@@ -528,7 +577,9 @@ def classify(outcome: SourceOutcome) -> _Classified:
):
notes.append("payload marked degraded")
expected = _EXPECTED.get(outcome.source)
- if count == 0:
+ if extract_error and extract_error.startswith(_RAISED):
+ notes.append(extract_error[:150]) # in full in extract_error
+ elif count == 0:
notes.append(
"payload has no records: a quiet period and a silent upstream "
"failure look the same here"
@@ -1300,7 +1351,8 @@ class HistoryStore:
"ok": "last capture was live and complete: records, every upstream fetch "
"answered, no degradation marker",
"degraded": "last capture was live but degraded: marked degraded, empty, "
- "partial (fewer series than expected), or its extractor found nothing",
+ "partial (fewer series than expected), or its extractor found nothing or "
+ "raised",
"stale": "last capture replayed an expired cached copy because an upstream "
"fetch failed; that is not a success, so overdue counts from the last live "
"capture",
diff --git a/src/world_intel_mcp/tests/test_history_honesty.py b/src/world_intel_mcp/tests/test_history_honesty.py
index 8b3462e..e251411 100644
--- a/src/world_intel_mcp/tests/test_history_honesty.py
+++ b/src/world_intel_mcp/tests/test_history_honesty.py
@@ -1021,3 +1021,86 @@ def test_a_file_that_is_not_a_database_raises_and_leaks_no_connection(tmp_path,
with pytest.raises(sqlite3.ProgrammingError, match="closed"):
conn.execute("SELECT 1")
assert [w for w in caught if issubclass(w.category, ResourceWarning)] == []
+
+
+# ---------------------------------------------------------------------------
+# M4: a raising extractor degrades the capture. L2: quote units.
+# ---------------------------------------------------------------------------
+
+
+def test_a_raising_extractor_degrades_the_capture_and_keeps_it(store, monkeypatch) -> None:
+ monkeypatch.setattr(history, "_EXTRACTORS", dict(history._EXTRACTORS))
+ monkeypatch.setattr(history, "_EXPECTED", dict(history._EXPECTED))
+ history.register_extractor("test_feed", lambda payload: payload["missing"])
+ _record(store, "test_feed", T0, {"rows": [1]})
+ (cap,) = store.capture_history("test_feed")["captures"]
+ assert cap["status"] == "degraded"
+ assert cap["reason"] == cap["extract_error"] == "extractor raised: KeyError: 'missing'"
+ assert (cap["record_count"], cap["observations_extracted"]) == (1, None)
+ replay = store.capture_history("test_feed", capture_id=cap["capture_id"])["capture"]
+ assert replay["payload"] == {"rows": [1]}
+ (row,) = store.collection_health(["test_feed"], now=T0)["sources"]
+ assert (row["state"], row["consecutive_failures"]) == ("degraded", 0)
+ assert (row["last_success_at"], row["last_ok_at"]) == (cap["collected_at"], None)
+ assert row["last_reason"] == row["last_extract_error"] == cap["reason"]
+
+ def _long(payload):
+ raise ValueError("x" * 300)
+
+ history.register_extractor("test_feed", _long, replace=True)
+ _record(store, "test_feed", T0 + 300, {"rows": [1]})
+ newest = store.capture_history("test_feed")["captures"][0]
+ assert newest["extract_error"] == "extractor raised: ValueError: " + "x" * 200
+ assert newest["reason"] == newest["extract_error"][:150]
+
+
+def test_a_raising_quote_extractor_is_not_called_partial(store, monkeypatch) -> None:
+ monkeypatch.setattr(history, "_EXTRACTORS", dict(history._EXTRACTORS))
+ monkeypatch.setattr(history, "_EXPECTED", dict(history._EXPECTED))
+
+ def _broken(payload):
+ raise TypeError("quotes moved")
+
+ history.register_extractor("market_quotes", _broken, replace=True, expected=7)
+ _record(store, "market_quotes", T0, _quotes(1.0, 2.0))
+ (cap,) = store.capture_history("market_quotes")["captures"]
+ assert (cap["status"], cap["reason"]) == ("degraded", "extractor raised: TypeError: quotes moved")
+
+
+def test_the_builtin_kp_extractor_raising_on_list_rows_degrades_the_capture(store) -> None:
+ """Upstream Kp rows that turn from objects into lists make the built-in
+ extractor raise; that capture is degraded, not a healthy ok."""
+ _record(store, "space_weather", T0, {"kp_recent": [["2026-10-16T21:00:00Z", 2.33]], "alerts": []})
+ (cap,) = store.capture_history("space_weather")["captures"]
+ assert (cap["status"], cap["reason"]) == (
+ "degraded", "extractor raised: AttributeError: 'list' object has no attribute 'get'"
+ )
+ assert (cap["extract_error"], cap["observations_extracted"]) == (cap["reason"], None)
+
+
+def test_quote_units_say_what_the_number_is() -> None:
+ assert set(history.COMMODITY_UNITS) == set(history._COMMODITY_SYMBOLS)
+ quotes = {"quotes": [
+ {"symbol": "^GSPC", "price": 6_123.5, "currency": "USD"},
+ {"symbol": "^N225", "price": 48_000.0, "currency": "JPY"},
+ {"symbol": "^HSI", "price": 26_000.0},
+ {"symbol": "^TNX", "price": 4.1, "currency": "USD"},
+ {"symbol": "AAPL", "price": 250.0, "currency": "USD"},
+ ]}
+ points = {p.series_key: (p.unit, p.label) for p in history._EXTRACTORS["market_quotes"](quotes)}
+ assert points == {
+ "^GSPC:price": ("index points", "^GSPC (USD)"),
+ "^N225:price": ("index points", "^N225 (JPY)"),
+ "^HSI:price": ("index points", "^HSI"),
+ "^TNX:price": (None, "^TNX"),
+ "AAPL:price": ("USD", "AAPL"),
+ }
+ commodities = {"commodities": [
+ {"symbol": "ZC=F", "name": "Corn", "price": 420.25},
+ {"symbol": "XX=F", "price": 1.0},
+ ]}
+ points = {
+ p.series_key: (p.unit, p.label)
+ for p in history._EXTRACTORS["commodity_quotes"](commodities)
+ }
+ assert points == {"ZC=F:price": ("US cents per bushel", "Corn"), "XX=F:price": (None, "XX=F")}