"Vibe-Trading: Your Personal Trading Agent"
Python ai-agentalgorithmic-tradingbacktestingfintechllmmcpmulti-agentpythonquantitative-financetrading
fix(eastmoney): decode JSONP news bodies and surface degraded indexes (#2081)
Jackzigen5a442c328252
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex bcf1ea3d3..34ce2f85b 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -7,6 +7,13 @@ This project adheres to [Semantic Versioning](https://semver.org/).### Fixed+- **JSONP-wrapped provider bodies** decode instead of raising: the throttled+ JSON GET helper unwraps a `callback(...)` envelope when `response.json()`+ cannot parse the body, so A-share news search keeps working when Eastmoney+ switches to JSONP (#2081).+- **Degraded Eastmoney news indexes** now raise an actionable error instead of+ reading as an empty article list: the Guba user index (`passportWeb`) can no+ longer masquerade as "this stock has no news" (#2081).- **Global futures routing and pricing**: digit-bearing contracts such as`M2K=F` and `6EZ2026.CME`, including `local:` inputs, reach the globalfutures engine with their full product code. OJ and ZR use quote-unitdiff --git a/agent/backtest/loaders/_http.py b/agent/backtest/loaders/_http.pyindex 89fe6ab42..9443d629f 100644--- a/agent/backtest/loaders/_http.py+++ b/agent/backtest/loaders/_http.py@@ -18,8 +18,10 @@ machines. For batch jobs raise the relevant ``*_MIN_INTERVAL`` env var.from __future__ import annotations+import jsonimport loggingimport random+import reimport threadingimport timefrom typing import Any@@ -173,6 +175,46 @@ def throttled_get(return session.get(url, params=params, headers=merged_headers, timeout=timeout)+# A JSONP envelope is a JS callback identifier followed by a parenthesized body,+# optionally terminated by ';'. The identifier is restricted to legal JS names+# (incl. dotted namespaces like "jQuery.cb") so a plain JSON array/object body —+# which never starts this way — is left untouched.+_JSONP_WRAPPER = re.compile(+ r"^[A-Za-z_$][\w$]*(?:\.[A-Za-z_$][\w$]*)*\s*\((.*)\)\s*;?\s*$",+ re.DOTALL,+)+++def decode_jsonp_body(text: str) -> Any:+ """Decode a body that may be bare JSON or a JSONP ``callback(...)`` envelope.++ Free provider surfaces sometimes answer with a JSONP envelope rather than+ JSON — Eastmoney's news search does so whenever it distrusts the caller —+ and ``response.json()`` cannot parse that. Plain JSON is decoded as-is+ first, so parentheses inside string values (article titles are full of+ them) are never mistaken for an envelope.++ Args:+ text: Raw response body.++ Returns:+ The decoded payload, or ``None`` when the body is neither JSON nor a+ JSONP-wrapped JSON value.+ """+ stripped = text.strip()+ try:+ return json.loads(stripped)+ except ValueError:+ pass+ match = _JSONP_WRAPPER.match(stripped)+ if match is None:+ return None+ try:+ return json.loads(match.group(1))+ except ValueError:+ return None++def throttled_get_json(url: str,*,@@ -187,6 +229,10 @@ def throttled_get_json(Same contract as :func:`throttled_get`, plus ``response.raise_for_status()``and ``response.json()``. A non-2xx status or undecodable body raises, whichthe caller's bounded-retry wrapper treats as transient.++ A body that fails to parse as JSON is retried as a JSONP envelope (see+ :func:`decode_jsonp_body`); anything else re-raises the original error, so a+ genuinely malformed body keeps its old failure mode."""response = throttled_get(url,@@ -197,7 +243,13 @@ def throttled_get_json(timeout=timeout,)response.raise_for_status()- return response.json()+ try:+ return response.json()+ except ValueError:+ decoded = decode_jsonp_body(response.text)+ if decoded is None:+ raise+ return decodeddef throttled_post_json(diff --git a/agent/backtest/loaders/eastmoney_client.py b/agent/backtest/loaders/eastmoney_client.pyindex 67093a131..d17d435f6 100644--- a/agent/backtest/loaders/eastmoney_client.py+++ b/agent/backtest/loaders/eastmoney_client.py@@ -20,12 +20,14 @@ Eastmoney addresses every instrument by a ``secid`` of the form ``<market>.<codefrom __future__ import annotations-import jsonimport logging-import refrom typing import Any-from backtest.loaders._http import resolve_min_interval, throttled_get_json+from backtest.loaders._http import (+ decode_jsonp_body,+ resolve_min_interval,+ throttled_get_json,+)logger = logging.getLogger(__name__)@@ -63,15 +65,6 @@ _FIELDS2 = "f51,f52,f53,f54,f55,f56,f57"# the search endpoint. Keyed by the upper-cased bare ticker (e.g. "AAPL")._US_SECID_CACHE: dict[str, str | None] = {}-# A JSONP envelope is a JS callback identifier followed by a parenthesized body,-# optionally terminated by ';'. The identifier is restricted to legal JS names-# (incl. dotted namespaces like "jQuery.cb") so a plain JSON array/object body —-# which never starts this way — is left untouched.-_JSONP_WRAPPER = re.compile(- r"^[A-Za-z_$][\w$]*(?:\.[A-Za-z_$][\w$]*)*\s*\((.*)\)\s*;?\s*$",- re.DOTALL,-)-def datacenter_rejection(payload: Any) -> str | None:"""Return the datacenter's complaint when it rejected a report query.@@ -181,11 +174,8 @@ def _parse_us_secid(payload: Any) -> str | None:def _strip_jsonp(text: str) -> Any:"""Decode a possibly JSONP-wrapped body to a Python object.- The suggest endpoint sometimes wraps its JSON in a ``callback(...)`` envelope.- A plain JSON body is parsed as-is first; only a body that actually matches a- leading callback identifier is unwrapped. This avoids corrupting plain JSON- whose string values contain parentheses (an earlier first-``(`` / last-``)``- slice mangled such bodies and returned ``None``).+ Delegates to :func:`backtest.loaders._http.decode_jsonp_body`, the single+ implementation shared with :func:`backtest.loaders._http.throttled_get_json`.Args:text: Raw response body.@@ -193,18 +183,7 @@ def _strip_jsonp(text: str) -> Any:Returns:The decoded object, or ``None`` when nothing parseable is found."""- stripped = text.strip()- try:- return json.loads(stripped)- except (ValueError, TypeError):- pass- match = _JSONP_WRAPPER.match(stripped)- if match is None:- return None- try:- return json.loads(match.group(1).strip())- except (ValueError, TypeError):- return None+ return decode_jsonp_body(text)def _resolve_us_secid(code: str) -> str | None:diff --git a/agent/src/tools/stock_news_tool.py b/agent/src/tools/stock_news_tool.pyindex 24efdcb35..c25471dbe 100644--- a/agent/src/tools/stock_news_tool.py+++ b/agent/src/tools/stock_news_tool.py@@ -161,7 +161,8 @@ def _fetch_eastmoney_news(query: str, limit: int) -> list[dict[str, Any]]:Raises:requests.RequestException: Network failure, propagated to the caller.requests.HTTPError: Non-2xx response status.- ValueError: Body is not valid JSON.+ ValueError: Body is not valid JSON, or Eastmoney answered with its+ degraded user index instead of the CMS article index."""param = json.dumps({@@ -193,6 +194,16 @@ def _fetch_eastmoney_news(query: str, limit: int) -> list[dict[str, Any]]:return []articles = result.get("cmsArticleWebOld")if not isinstance(articles, list):+ # Eastmoney swaps the CMS article index for its Guba user index+ # ("passportWeb") when it distrusts the caller, so a degraded answer is+ # shaped like a valid but empty one. Reading it as ``[]`` would turn a+ # blocked upstream into a silent "no news for this stock" (#2081).+ if result:+ raise ValueError(+ "eastmoney returned no cmsArticleWebOld section "+ f"(result keys: {sorted(result)}): the search index was degraded,"+ " most likely anti-bot filtering of this network"+ )return []return [_em_article(a) for a in articles if isinstance(a, dict)][:limit]diff --git a/agent/tests/test_http_jsonp_fallback.py b/agent/tests/test_http_jsonp_fallback.pynew file mode 100644index 000000000..6f7b20e4b--- /dev/null+++ b/agent/tests/test_http_jsonp_fallback.py@@ -0,0 +1,67 @@+"""Tests for the JSONP fallback in the throttled JSON GET helper.++Eastmoney's search endpoints answer with a ``callback(...)`` envelope instead of+a JSON body when they distrust the caller (see issue #2081), which+``response.json()`` cannot parse. These tests pin the fallback at the HTTP+boundary so every caller of :func:`backtest.loaders._http.throttled_get_json`+benefits, and pin that a genuinely malformed body keeps raising.++No request leaves the process: :func:`backtest.loaders._http.throttled_get` is+mocked with a stand-in response object.+"""++from __future__ import annotations++import json+from typing import Any+from unittest.mock import patch++import pytest++from backtest.loaders import _http+++class _FakeResponse:+ """Minimal stand-in for ``requests.Response`` at the HTTP boundary."""++ def __init__(self, body: str) -> None:+ self.text = body++ def raise_for_status(self) -> None:+ """No-op: these tests never model a non-2xx status."""++ def json(self) -> Any:+ """Mimic ``requests``: undecodable bodies raise ``ValueError``."""+ return json.loads(self.text)+++def _call(body: str) -> Any:+ """Run ``throttled_get_json`` against a canned response body."""+ with patch.object(_http, "throttled_get", return_value=_FakeResponse(body)):+ return _http.throttled_get_json(+ "https://search-api-web.eastmoney.com/search/jsonp",+ host_key="eastmoney",+ min_interval=0,+ params={"cb": ""},+ )+++class TestJsonpFallback:+ def test_jsonp_envelope_is_decoded(self) -> None:+ body = 'jQuery3700123_456({"code": 0, "result": {"hitsTotal": 1}})'+ assert _call(body) == {"code": 0, "result": {"hitsTotal": 1}}++ def test_dotted_callback_identifier_is_decoded(self) -> None:+ body = 'window.jQuery.cb({"code": 0});'+ assert _call(body) == {"code": 0}++ def test_plain_json_still_parses(self) -> None:+ assert _call('{"code": 0, "result": {}}') == {"code": 0, "result": {}}++ def test_plain_json_with_parens_in_values_is_not_mangled(self) -> None:+ body = '{"title": "公司(600519)发布三季报(更新)"}'+ assert _call(body) == {"title": "公司(600519)发布三季报(更新)"}++ def test_unparseable_body_still_raises(self) -> None:+ with pytest.raises(ValueError):+ _call("<html>blocked</html>")diff --git a/agent/tests/test_stock_news_tool.py b/agent/tests/test_stock_news_tool.pyindex 1e8a176b9..f5a11d3f8 100644--- a/agent/tests/test_stock_news_tool.py+++ b/agent/tests/test_stock_news_tool.py@@ -243,3 +243,36 @@ class TestExecuteError:assert out["ok"] is Falseassert "yahoo 429" in out["error"]assert "yahoo news fetch failed" in out["error"]+++class TestEastmoneyDegradedIndex:+ """A degraded Eastmoney answer must not read as "this stock has no news".++ Eastmoney serves its Guba user index (``passportWeb``) instead of the CMS+ article index when it distrusts the caller, and that degraded answer is+ shaped like a valid but empty one (see issue #2081).+ """++ def test_degraded_user_index_raises_instead_of_empty(self) -> None:+ payload = {"code": 0, "result": {"passportWeb": [], "userWeb": []}}+ with patch.object(eastmoney_client, "throttled_get_json", return_value=payload):+ out = json.loads(StockNewsTool().execute(code="600519.SH"))++ assert out["ok"] is False+ assert "degraded" in out["error"]++ def test_empty_cms_section_is_a_clean_empty_result(self) -> None:+ payload = {"code": 0, "result": {"cmsArticleWebOld": []}}+ with patch.object(eastmoney_client, "throttled_get_json", return_value=payload):+ out = json.loads(StockNewsTool().execute(code="600519.SH"))++ assert out["ok"] is True+ assert out["data"]["articles"] == []++ def test_jsonp_enveloped_body_is_parsed_into_articles(self) -> None:+ body = "jQuery3700123_456(" + json.dumps(_em_news_payload(), ensure_ascii=False) + ")"+ with patch.object(eastmoney_client, "throttled_get_json", return_value=body):+ out = json.loads(StockNewsTool().execute(code="600519.SH"))++ assert out["ok"] is True+ assert len(out["data"]["articles"]) == 2