Python
fix(yahoo): read a news article whose fields are null (#1458)
Yijia-Xiao9dceea6e9d32
diff --git a/tests/test_news_lookahead.py b/tests/test_news_lookahead.pyindex 9d96dfbab..366605075 100644--- a/tests/test_news_lookahead.py+++ b/tests/test_news_lookahead.py@@ -255,3 +255,33 @@ def test_the_article_limit_still_caps_what_is_returned(monkeypatch):out = ynews.get_global_news_yfinance("2025-05-09", look_back_days=7, limit=3)assert out.count("### ") == 3++[email protected]("field", ["title", "summary", "provider", "canonicalUrl", "pubDate"])+def test_an_article_with_a_null_field_is_read_with_its_default(field):+ """Yahoo sends some fields as null rather than leaving them out (#1458)."""+ content = {"title": "T", "summary": "S", "provider": {"displayName": "P"},+ "canonicalUrl": {"url": "https://x"}, "pubDate": "2026-09-22T10:00:00Z", field: None}+ data = ynews._extract_article_data({"content": content})+ assert isinstance(data["title"], str) and isinstance(data["summary"], str)+ assert isinstance(data["publisher"], str) and isinstance(data["link"], str)++[email protected]("field", ["title", "summary", "publisher", "link"])+def test_a_flat_article_with_a_null_field_is_read_with_its_default(field):+ data = ynews._extract_article_data({"title": "T", "summary": "S", "publisher": "P", "link": "L",+ "providerPublishTime": _epoch("2026-09-22"), field: None})+ assert all(isinstance(data[k], str) for k in ("title", "summary", "publisher", "link"))+++def test_one_malformed_article_does_not_lose_the_feed(monkeypatch):+ good = {"content": {"title": "Kept", "summary": "", "provider": {"displayName": "Wire"},+ "pubDate": "2026-09-22T10:00:00Z"}}+ bad = {"content": {"title": "Also kept", "summary": None, "provider": None,+ "pubDate": "2026-09-22T11:00:00Z"}}+ _ticker_with([good, bad], monkeypatch)+ out = ynews.get_news_yfinance("ZS", "2026-09-20", "2026-09-23")+ assert "Kept" in out and "Also kept" in outdiff --git a/tradingagents/dataflows/vendors/yahoo/news.py b/tradingagents/dataflows/vendors/yahoo/news.pyindex 1671daa8b..7328beca7 100644--- a/tradingagents/dataflows/vendors/yahoo/news.py+++ b/tradingagents/dataflows/vendors/yahoo/news.py@@ -13,48 +13,44 @@ from tradingagents.dataflows.vendors.yahoo.common import yf_retrydef _extract_article_data(article: dict) -> dict:- """Extract article data from yfinance news format (handles nested 'content' structure)."""- if "content" in article:- content = article["content"]- title = content.get("title", "No title")- summary = content.get("summary", "")- provider = content.get("provider", {})- publisher = provider.get("displayName", "Unknown")+ """Extract article data from yfinance news format (handles nested 'content' structure).+ Yahoo sends some fields as null rather than leaving them out, so each falls+ back to its default on a null as well as on a missing key (#1458).+ """+ content = article.get("content")+ if isinstance(content, dict):url_obj = content.get("canonicalUrl") or content.get("clickThroughUrl") or {}- link = url_obj.get("url", "")-- pub_date_str = content.get("pubDate", "")+ pub_date_str = content.get("pubDate") or ""pub_date = Noneif pub_date_str:with contextlib.suppress(ValueError, AttributeError):pub_date = datetime.fromisoformat(pub_date_str.replace("Z", "+00:00"))return {- "title": title,- "summary": summary,- "publisher": publisher,- "link": link,- "pub_date": pub_date,- }- else:- # Fallback for flat structure. Parse the epoch publish time so flat- # articles are date-filterable too (otherwise they bypass the- # historical window and leak future news, #992/#1007).- pub_date = None- ts = article.get("providerPublishTime")- if ts:- # Epoch seconds are UTC; parse them as UTC-aware so filtering does- # not shift with the host timezone (#1126).- with contextlib.suppress(ValueError, OSError, TypeError):- pub_date = datetime.fromtimestamp(ts, tz=UTC)- return {- "title": article.get("title", "No title"),- "summary": article.get("summary", ""),- "publisher": article.get("publisher", "Unknown"),- "link": article.get("link", ""),+ "title": content.get("title") or "No title",+ "summary": content.get("summary") or "",+ "publisher": (content.get("provider") or {}).get("displayName") or "Unknown",+ "link": url_obj.get("url") or "","pub_date": pub_date,}+ # Fallback for flat structure. Parse the epoch publish time so flat+ # articles are date-filterable too (otherwise they bypass the+ # historical window and leak future news, #992/#1007).+ pub_date = None+ ts = article.get("providerPublishTime")+ if ts:+ # Epoch seconds are UTC; parse them as UTC-aware so filtering does+ # not shift with the host timezone (#1126).+ with contextlib.suppress(ValueError, OSError, TypeError):+ pub_date = datetime.fromtimestamp(ts, tz=UTC)+ return {+ "title": article.get("title") or "No title",+ "summary": article.get("summary") or "",+ "publisher": article.get("publisher") or "Unknown",+ "link": article.get("link") or "",+ "pub_date": pub_date,+ }def get_news_yfinance(