Skip to content
Python

fix(yahoo): read a news article whose fields are null (#1458)

Yijia-Xiao9dceea6e9d32
diff --git a/tests/test_news_lookahead.py b/tests/test_news_lookahead.py
index 9d96dfbab..366605075 100644
--- a/tests/test_news_lookahead.py
+++ b/tests/test_news_lookahead.py
@@ -255,3 +255,33 @@ def test_the_article_limit_still_caps_what_is_returned(monkeypatch):
out = ynews.get_global_news_yfinance("2025-05-09", look_back_days=7, limit=3)
assert out.count("### ") == 3
+
+
[email protected]("field", ["title", "summary", "provider", "canonicalUrl", "pubDate"])
+def test_an_article_with_a_null_field_is_read_with_its_default(field):
+ """Yahoo sends some fields as null rather than leaving them out (#1458)."""
+ content = {"title": "T", "summary": "S", "provider": {"displayName": "P"},
+ "canonicalUrl": {"url": "https://x"}, "pubDate": "2026-09-22T10:00:00Z", field: None}
+ data = ynews._extract_article_data({"content": content})
+ assert isinstance(data["title"], str) and isinstance(data["summary"], str)
+ assert isinstance(data["publisher"], str) and isinstance(data["link"], str)
+
+
[email protected]("field", ["title", "summary", "publisher", "link"])
+def test_a_flat_article_with_a_null_field_is_read_with_its_default(field):
+ data = ynews._extract_article_data({"title": "T", "summary": "S", "publisher": "P", "link": "L",
+ "providerPublishTime": _epoch("2026-09-22"), field: None})
+ assert all(isinstance(data[k], str) for k in ("title", "summary", "publisher", "link"))
+
+
+def test_one_malformed_article_does_not_lose_the_feed(monkeypatch):
+ good = {"content": {"title": "Kept", "summary": "", "provider": {"displayName": "Wire"},
+ "pubDate": "2026-09-22T10:00:00Z"}}
+ bad = {"content": {"title": "Also kept", "summary": None, "provider": None,
+ "pubDate": "2026-09-22T11:00:00Z"}}
+ _ticker_with([good, bad], monkeypatch)
+ out = ynews.get_news_yfinance("ZS", "2026-09-20", "2026-09-23")
+ assert "Kept" in out and "Also kept" in out
diff --git a/tradingagents/dataflows/vendors/yahoo/news.py b/tradingagents/dataflows/vendors/yahoo/news.py
index 1671daa8b..7328beca7 100644
--- a/tradingagents/dataflows/vendors/yahoo/news.py
+++ b/tradingagents/dataflows/vendors/yahoo/news.py
@@ -13,48 +13,44 @@ from tradingagents.dataflows.vendors.yahoo.common import yf_retry
def _extract_article_data(article: dict) -> dict:
- """Extract article data from yfinance news format (handles nested 'content' structure)."""
- if "content" in article:
- content = article["content"]
- title = content.get("title", "No title")
- summary = content.get("summary", "")
- provider = content.get("provider", {})
- publisher = provider.get("displayName", "Unknown")
+ """Extract article data from yfinance news format (handles nested 'content' structure).
+ Yahoo sends some fields as null rather than leaving them out, so each falls
+ back to its default on a null as well as on a missing key (#1458).
+ """
+ content = article.get("content")
+ if isinstance(content, dict):
url_obj = content.get("canonicalUrl") or content.get("clickThroughUrl") or {}
- link = url_obj.get("url", "")
-
- pub_date_str = content.get("pubDate", "")
+ pub_date_str = content.get("pubDate") or ""
pub_date = None
if pub_date_str:
with contextlib.suppress(ValueError, AttributeError):
pub_date = datetime.fromisoformat(pub_date_str.replace("Z", "+00:00"))
return {
- "title": title,
- "summary": summary,
- "publisher": publisher,
- "link": link,
- "pub_date": pub_date,
- }
- else:
- # Fallback for flat structure. Parse the epoch publish time so flat
- # articles are date-filterable too (otherwise they bypass the
- # historical window and leak future news, #992/#1007).
- pub_date = None
- ts = article.get("providerPublishTime")
- if ts:
- # Epoch seconds are UTC; parse them as UTC-aware so filtering does
- # not shift with the host timezone (#1126).
- with contextlib.suppress(ValueError, OSError, TypeError):
- pub_date = datetime.fromtimestamp(ts, tz=UTC)
- return {
- "title": article.get("title", "No title"),
- "summary": article.get("summary", ""),
- "publisher": article.get("publisher", "Unknown"),
- "link": article.get("link", ""),
+ "title": content.get("title") or "No title",
+ "summary": content.get("summary") or "",
+ "publisher": (content.get("provider") or {}).get("displayName") or "Unknown",
+ "link": url_obj.get("url") or "",
"pub_date": pub_date,
}
+ # Fallback for flat structure. Parse the epoch publish time so flat
+ # articles are date-filterable too (otherwise they bypass the
+ # historical window and leak future news, #992/#1007).
+ pub_date = None
+ ts = article.get("providerPublishTime")
+ if ts:
+ # Epoch seconds are UTC; parse them as UTC-aware so filtering does
+ # not shift with the host timezone (#1126).
+ with contextlib.suppress(ValueError, OSError, TypeError):
+ pub_date = datetime.fromtimestamp(ts, tz=UTC)
+ return {
+ "title": article.get("title") or "No title",
+ "summary": article.get("summary") or "",
+ "publisher": article.get("publisher") or "Unknown",
+ "link": article.get("link") or "",
+ "pub_date": pub_date,
+ }
def get_news_yfinance(