Skip to content

"Vibe-Trading: Your Personal Trading Agent"

fix(eastmoney): decode JSONP news bodies and surface degraded indexes (#2081)

Jackzigen5a442c328252
diff --git a/CHANGELOG.md b/CHANGELOG.md
index bcf1ea3d3..34ce2f85b 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -7,6 +7,13 @@ This project adheres to [Semantic Versioning](https://semver.org/).
### Fixed
+- **JSONP-wrapped provider bodies** decode instead of raising: the throttled
+ JSON GET helper unwraps a `callback(...)` envelope when `response.json()`
+ cannot parse the body, so A-share news search keeps working when Eastmoney
+ switches to JSONP (#2081).
+- **Degraded Eastmoney news indexes** now raise an actionable error instead of
+ reading as an empty article list: the Guba user index (`passportWeb`) can no
+ longer masquerade as "this stock has no news" (#2081).
- **Global futures routing and pricing**: digit-bearing contracts such as
`M2K=F` and `6EZ2026.CME`, including `local:` inputs, reach the global
futures engine with their full product code. OJ and ZR use quote-unit
diff --git a/agent/backtest/loaders/_http.py b/agent/backtest/loaders/_http.py
index 89fe6ab42..9443d629f 100644
--- a/agent/backtest/loaders/_http.py
+++ b/agent/backtest/loaders/_http.py
@@ -18,8 +18,10 @@ machines. For batch jobs raise the relevant ``*_MIN_INTERVAL`` env var.
from __future__ import annotations
+import json
import logging
import random
+import re
import threading
import time
from typing import Any
@@ -173,6 +175,46 @@ def throttled_get(
return session.get(url, params=params, headers=merged_headers, timeout=timeout)
+# A JSONP envelope is a JS callback identifier followed by a parenthesized body,
+# optionally terminated by ';'. The identifier is restricted to legal JS names
+# (incl. dotted namespaces like "jQuery.cb") so a plain JSON array/object body —
+# which never starts this way — is left untouched.
+_JSONP_WRAPPER = re.compile(
+ r"^[A-Za-z_$][\w$]*(?:\.[A-Za-z_$][\w$]*)*\s*\((.*)\)\s*;?\s*$",
+ re.DOTALL,
+)
+
+
+def decode_jsonp_body(text: str) -> Any:
+ """Decode a body that may be bare JSON or a JSONP ``callback(...)`` envelope.
+
+ Free provider surfaces sometimes answer with a JSONP envelope rather than
+ JSON — Eastmoney's news search does so whenever it distrusts the caller —
+ and ``response.json()`` cannot parse that. Plain JSON is decoded as-is
+ first, so parentheses inside string values (article titles are full of
+ them) are never mistaken for an envelope.
+
+ Args:
+ text: Raw response body.
+
+ Returns:
+ The decoded payload, or ``None`` when the body is neither JSON nor a
+ JSONP-wrapped JSON value.
+ """
+ stripped = text.strip()
+ try:
+ return json.loads(stripped)
+ except ValueError:
+ pass
+ match = _JSONP_WRAPPER.match(stripped)
+ if match is None:
+ return None
+ try:
+ return json.loads(match.group(1))
+ except ValueError:
+ return None
+
+
def throttled_get_json(
url: str,
*,
@@ -187,6 +229,10 @@ def throttled_get_json(
Same contract as :func:`throttled_get`, plus ``response.raise_for_status()``
and ``response.json()``. A non-2xx status or undecodable body raises, which
the caller's bounded-retry wrapper treats as transient.
+
+ A body that fails to parse as JSON is retried as a JSONP envelope (see
+ :func:`decode_jsonp_body`); anything else re-raises the original error, so a
+ genuinely malformed body keeps its old failure mode.
"""
response = throttled_get(
url,
@@ -197,7 +243,13 @@ def throttled_get_json(
timeout=timeout,
)
response.raise_for_status()
- return response.json()
+ try:
+ return response.json()
+ except ValueError:
+ decoded = decode_jsonp_body(response.text)
+ if decoded is None:
+ raise
+ return decoded
def throttled_post_json(
diff --git a/agent/backtest/loaders/eastmoney_client.py b/agent/backtest/loaders/eastmoney_client.py
index 67093a131..d17d435f6 100644
--- a/agent/backtest/loaders/eastmoney_client.py
+++ b/agent/backtest/loaders/eastmoney_client.py
@@ -20,12 +20,14 @@ Eastmoney addresses every instrument by a ``secid`` of the form ``<market>.<code
from __future__ import annotations
-import json
import logging
-import re
from typing import Any
-from backtest.loaders._http import resolve_min_interval, throttled_get_json
+from backtest.loaders._http import (
+ decode_jsonp_body,
+ resolve_min_interval,
+ throttled_get_json,
+)
logger = logging.getLogger(__name__)
@@ -63,15 +65,6 @@ _FIELDS2 = "f51,f52,f53,f54,f55,f56,f57"
# the search endpoint. Keyed by the upper-cased bare ticker (e.g. "AAPL").
_US_SECID_CACHE: dict[str, str | None] = {}
-# A JSONP envelope is a JS callback identifier followed by a parenthesized body,
-# optionally terminated by ';'. The identifier is restricted to legal JS names
-# (incl. dotted namespaces like "jQuery.cb") so a plain JSON array/object body —
-# which never starts this way — is left untouched.
-_JSONP_WRAPPER = re.compile(
- r"^[A-Za-z_$][\w$]*(?:\.[A-Za-z_$][\w$]*)*\s*\((.*)\)\s*;?\s*$",
- re.DOTALL,
-)
-
def datacenter_rejection(payload: Any) -> str | None:
"""Return the datacenter's complaint when it rejected a report query.
@@ -181,11 +174,8 @@ def _parse_us_secid(payload: Any) -> str | None:
def _strip_jsonp(text: str) -> Any:
"""Decode a possibly JSONP-wrapped body to a Python object.
- The suggest endpoint sometimes wraps its JSON in a ``callback(...)`` envelope.
- A plain JSON body is parsed as-is first; only a body that actually matches a
- leading callback identifier is unwrapped. This avoids corrupting plain JSON
- whose string values contain parentheses (an earlier first-``(`` / last-``)``
- slice mangled such bodies and returned ``None``).
+ Delegates to :func:`backtest.loaders._http.decode_jsonp_body`, the single
+ implementation shared with :func:`backtest.loaders._http.throttled_get_json`.
Args:
text: Raw response body.
@@ -193,18 +183,7 @@ def _strip_jsonp(text: str) -> Any:
Returns:
The decoded object, or ``None`` when nothing parseable is found.
"""
- stripped = text.strip()
- try:
- return json.loads(stripped)
- except (ValueError, TypeError):
- pass
- match = _JSONP_WRAPPER.match(stripped)
- if match is None:
- return None
- try:
- return json.loads(match.group(1).strip())
- except (ValueError, TypeError):
- return None
+ return decode_jsonp_body(text)
def _resolve_us_secid(code: str) -> str | None:
diff --git a/agent/src/tools/stock_news_tool.py b/agent/src/tools/stock_news_tool.py
index 24efdcb35..c25471dbe 100644
--- a/agent/src/tools/stock_news_tool.py
+++ b/agent/src/tools/stock_news_tool.py
@@ -161,7 +161,8 @@ def _fetch_eastmoney_news(query: str, limit: int) -> list[dict[str, Any]]:
Raises:
requests.RequestException: Network failure, propagated to the caller.
requests.HTTPError: Non-2xx response status.
- ValueError: Body is not valid JSON.
+ ValueError: Body is not valid JSON, or Eastmoney answered with its
+ degraded user index instead of the CMS article index.
"""
param = json.dumps(
{
@@ -193,6 +194,16 @@ def _fetch_eastmoney_news(query: str, limit: int) -> list[dict[str, Any]]:
return []
articles = result.get("cmsArticleWebOld")
if not isinstance(articles, list):
+ # Eastmoney swaps the CMS article index for its Guba user index
+ # ("passportWeb") when it distrusts the caller, so a degraded answer is
+ # shaped like a valid but empty one. Reading it as ``[]`` would turn a
+ # blocked upstream into a silent "no news for this stock" (#2081).
+ if result:
+ raise ValueError(
+ "eastmoney returned no cmsArticleWebOld section "
+ f"(result keys: {sorted(result)}): the search index was degraded,"
+ " most likely anti-bot filtering of this network"
+ )
return []
return [_em_article(a) for a in articles if isinstance(a, dict)][:limit]
diff --git a/agent/tests/test_http_jsonp_fallback.py b/agent/tests/test_http_jsonp_fallback.py
new file mode 100644
index 000000000..6f7b20e4b
--- /dev/null
+++ b/agent/tests/test_http_jsonp_fallback.py
@@ -0,0 +1,67 @@
+"""Tests for the JSONP fallback in the throttled JSON GET helper.
+
+Eastmoney's search endpoints answer with a ``callback(...)`` envelope instead of
+a JSON body when they distrust the caller (see issue #2081), which
+``response.json()`` cannot parse. These tests pin the fallback at the HTTP
+boundary so every caller of :func:`backtest.loaders._http.throttled_get_json`
+benefits, and pin that a genuinely malformed body keeps raising.
+
+No request leaves the process: :func:`backtest.loaders._http.throttled_get` is
+mocked with a stand-in response object.
+"""
+
+from __future__ import annotations
+
+import json
+from typing import Any
+from unittest.mock import patch
+
+import pytest
+
+from backtest.loaders import _http
+
+
+class _FakeResponse:
+ """Minimal stand-in for ``requests.Response`` at the HTTP boundary."""
+
+ def __init__(self, body: str) -> None:
+ self.text = body
+
+ def raise_for_status(self) -> None:
+ """No-op: these tests never model a non-2xx status."""
+
+ def json(self) -> Any:
+ """Mimic ``requests``: undecodable bodies raise ``ValueError``."""
+ return json.loads(self.text)
+
+
+def _call(body: str) -> Any:
+ """Run ``throttled_get_json`` against a canned response body."""
+ with patch.object(_http, "throttled_get", return_value=_FakeResponse(body)):
+ return _http.throttled_get_json(
+ "https://search-api-web.eastmoney.com/search/jsonp",
+ host_key="eastmoney",
+ min_interval=0,
+ params={"cb": ""},
+ )
+
+
+class TestJsonpFallback:
+ def test_jsonp_envelope_is_decoded(self) -> None:
+ body = 'jQuery3700123_456({"code": 0, "result": {"hitsTotal": 1}})'
+ assert _call(body) == {"code": 0, "result": {"hitsTotal": 1}}
+
+ def test_dotted_callback_identifier_is_decoded(self) -> None:
+ body = 'window.jQuery.cb({"code": 0});'
+ assert _call(body) == {"code": 0}
+
+ def test_plain_json_still_parses(self) -> None:
+ assert _call('{"code": 0, "result": {}}') == {"code": 0, "result": {}}
+
+ def test_plain_json_with_parens_in_values_is_not_mangled(self) -> None:
+ body = '{"title": "公司(600519)发布三季报(更新)"}'
+ assert _call(body) == {"title": "公司(600519)发布三季报(更新)"}
+
+ def test_unparseable_body_still_raises(self) -> None:
+ with pytest.raises(ValueError):
+ _call("<html>blocked</html>")
diff --git a/agent/tests/test_stock_news_tool.py b/agent/tests/test_stock_news_tool.py
index 1e8a176b9..f5a11d3f8 100644
--- a/agent/tests/test_stock_news_tool.py
+++ b/agent/tests/test_stock_news_tool.py
@@ -243,3 +243,36 @@ class TestExecuteError:
assert out["ok"] is False
assert "yahoo 429" in out["error"]
assert "yahoo news fetch failed" in out["error"]
+
+
+class TestEastmoneyDegradedIndex:
+ """A degraded Eastmoney answer must not read as "this stock has no news".
+
+ Eastmoney serves its Guba user index (``passportWeb``) instead of the CMS
+ article index when it distrusts the caller, and that degraded answer is
+ shaped like a valid but empty one (see issue #2081).
+ """
+
+ def test_degraded_user_index_raises_instead_of_empty(self) -> None:
+ payload = {"code": 0, "result": {"passportWeb": [], "userWeb": []}}
+ with patch.object(eastmoney_client, "throttled_get_json", return_value=payload):
+ out = json.loads(StockNewsTool().execute(code="600519.SH"))
+
+ assert out["ok"] is False
+ assert "degraded" in out["error"]
+
+ def test_empty_cms_section_is_a_clean_empty_result(self) -> None:
+ payload = {"code": 0, "result": {"cmsArticleWebOld": []}}
+ with patch.object(eastmoney_client, "throttled_get_json", return_value=payload):
+ out = json.loads(StockNewsTool().execute(code="600519.SH"))
+
+ assert out["ok"] is True
+ assert out["data"]["articles"] == []
+
+ def test_jsonp_enveloped_body_is_parsed_into_articles(self) -> None:
+ body = "jQuery3700123_456(" + json.dumps(_em_news_payload(), ensure_ascii=False) + ")"
+ with patch.object(eastmoney_client, "throttled_get_json", return_value=body):
+ out = json.loads(StockNewsTool().execute(code="600519.SH"))
+
+ assert out["ok"] is True
+ assert len(out["data"]["articles"]) == 2