Python
fix(yahoo): report a Yahoo request that failed as unavailable, not as no data
Yijia-Xiao389ff465072a
diff --git a/tests/test_fundamentals_lookahead.py b/tests/test_fundamentals_lookahead.pyindex d91d4390c..f39f88124 100644--- a/tests/test_fundamentals_lookahead.py+++ b/tests/test_fundamentals_lookahead.py@@ -125,10 +125,13 @@ class TestLiveRunUnchanged:@pytest.mark.unitclass TestNoUsableFieldsStillRaises:- def test_stub_payload_raises_no_market_data(self):+ def test_stub_payload_raises_no_market_data(self, monkeypatch):# yfinance returns {"trailingPegRatio": None} for unknown symbols; on a# live run that must stay a hard "no data", not a bare header.from tradingagents.dataflows.errors import NoMarketDataError+ from tradingagents.dataflows.vendors.yahoo import ohlcv++ monkeypatch.setattr(ohlcv, "vendor_reachable", lambda url: True)with pytest.raises(NoMarketDataError):_yf(_TODAY, info={"trailingPegRatio": None})diff --git a/tests/test_yahoo_rate_limit.py b/tests/test_yahoo_rate_limit.pyindex 97a6882ad..c441ef1b1 100644--- a/tests/test_yahoo_rate_limit.py+++ b/tests/test_yahoo_rate_limit.py@@ -21,7 +21,7 @@ from tradingagents.agents.tools import ()from tradingagents.dataflows import routerfrom tradingagents.dataflows.config import set_config-from tradingagents.dataflows.errors import NoMarketDataError, VendorRateLimitError+from tradingagents.dataflows.errors import VendorRateLimitErrorfrom tradingagents.dataflows.vendors.yahoo import fundamentals, ohlcvDAY = "2026-09-18"@@ -112,19 +112,67 @@ def test_fundamentals_ask_a_new_ticker_after_a_rate_limit(yahoo):@pytest.mark.unit-def test_another_price_fetch_error_still_tells_an_outage_from_an_unknown_symbol(yahoo):- """``Ticker.history`` lets some errors through that ``yf.download`` turned- into an empty frame. Whether Yahoo answers still decides which one it is."""+def test_a_price_request_that_raises_is_unavailable_whether_or_not_yahoo_answers(yahoo):+ """An unknown symbol comes back from ``Ticker.history`` as an empty frame, so+ a request that raised failed in transit, and says nothing about the symbol."""def refused(self, **kwargs):raise ConnectionError("curl: (7) Failed to connect to query2.finance.yahoo.com")yahoo.setattr(yf.Ticker, "history", refused)+ for reachable in (False, True):+ yahoo.setattr(ohlcv, "vendor_reachable", lambda url, _r=reachable: _r)+ with pytest.raises(VendorRateLimitError, match="request failed"):+ ohlcv.load_ohlcv("AAPL", DAY)- yahoo.setattr(ohlcv, "vendor_reachable", lambda url: False)- with pytest.raises(VendorRateLimitError, match="unreachable"):- ohlcv.load_ohlcv("AAPL", DAY)- yahoo.setattr(ohlcv, "vendor_reachable", lambda url: True)- with pytest.raises(NoMarketDataError):- ohlcv.load_ohlcv("AAPL", DAY)+def test_a_price_request_that_fails_is_unavailable_even_when_yahoo_answers(monkeypatch, tmp_path):+ """A timeout says nothing about the symbol; a reachable host does not make it one."""+ def timed_out(**kwargs):+ raise TimeoutError("read timed out")++ monkeypatch.setattr(ohlcv, "get_config", lambda: {"data_cache_dir": str(tmp_path)})+ monkeypatch.setattr(ohlcv.yf, "Ticker", lambda s: type("T", (), {"history": lambda self, **k: timed_out(**k)})())+ monkeypatch.setattr(ohlcv, "vendor_reachable", lambda url: True)++ with pytest.raises(VendorRateLimitError):+ ohlcv.load_ohlcv("NVDA", "2026-09-23")+++class _Unreachable:+ """A Ticker whose every request fails in transit."""++ def __init__(self, *args, **kwargs):+ pass++ def __getattr__(self, name):+ raise TimeoutError("read timed out")+++def _yahoo_calls():+ from tradingagents.dataflows.date_window import get_current_date+ from tradingagents.dataflows.vendors.yahoo import market, news++ today = get_current_date()+ return [+ pytest.param(lambda: fundamentals.get_fundamentals("AAPL", today), id="fundamentals"),+ pytest.param(lambda: fundamentals.get_balance_sheet("AAPL", "quarterly", today), id="statement"),+ pytest.param(lambda: fundamentals.get_insider_transactions("AAPL", today), id="insider"),+ pytest.param(lambda: fundamentals.get_company_profile("AAPL"), id="profile"),+ pytest.param(lambda: news.get_news_yfinance("AAPL", "2026-09-10", DAY), id="news"),+ pytest.param(lambda: news.get_global_news_yfinance(DAY, 7, 5), id="global_news"),+ pytest.param(lambda: market.get_YFin_data_online("AAPL", "2026-09-10", DAY), id="stock_data"),+ pytest.param(lambda: market.get_closes("AAPL", "2026-09-10", DAY), id="closes"),+ ]++[email protected]("call", _yahoo_calls())+def test_a_yahoo_request_that_fails_is_unavailable_not_absent(yahoo, call):+ """An unknown symbol comes back empty, so a request that raised failed in transit."""+ yahoo.setattr(yf, "Ticker", _Unreachable)+ yahoo.setattr(yf, "Search", _Unreachable)++ with pytest.raises(VendorRateLimitError):+ call()diff --git a/tradingagents/dataflows/vendors/yahoo/fundamentals.py b/tradingagents/dataflows/vendors/yahoo/fundamentals.pyindex 84d0e55c4..243c87704 100644--- a/tradingagents/dataflows/vendors/yahoo/fundamentals.py+++ b/tradingagents/dataflows/vendors/yahoo/fundamentals.py@@ -4,7 +4,7 @@ import pandas as pdimport yfinance as yffrom tradingagents.dataflows.date_window import withhold_live_profile-from tradingagents.dataflows.errors import NoMarketDataError, VendorError, VendorRateLimitError+from tradingagents.dataflows.errors import VendorRateLimitErrorfrom tradingagents.dataflows.net import vendor_reachablefrom tradingagents.dataflows.symbols import normalize_symbolfrom tradingagents.dataflows.vendors.yahoo.ohlcv import (@@ -32,60 +32,51 @@ def get_fundamentals(if withheld:return withheld- try:- info = yf_retry(lambda: yf.Ticker(canonical).info)-- if not info:- raise_for_empty(ticker, canonical, "fundamentals")-- fields = [- ("Name", info.get("longName")),- ("Sector", info.get("sector")),- ("Industry", info.get("industry")),- ("Market Cap", info.get("marketCap")),- ("PE Ratio (TTM)", info.get("trailingPE")),- ("Forward PE", info.get("forwardPE")),- ("PEG Ratio", info.get("pegRatio")),- ("Price to Book", info.get("priceToBook")),- ("EPS (TTM)", info.get("trailingEps")),- ("Forward EPS", info.get("forwardEps")),- ("Dividend Yield", info.get("dividendYield")),- ("Beta", info.get("beta")),- ("52 Week High", info.get("fiftyTwoWeekHigh")),- ("52 Week Low", info.get("fiftyTwoWeekLow")),- ("50 Day Average", info.get("fiftyDayAverage")),- ("200 Day Average", info.get("twoHundredDayAverage")),- ("Revenue (TTM)", info.get("totalRevenue")),- ("Gross Profit", info.get("grossProfits")),- ("EBITDA", info.get("ebitda")),- ("Net Income", info.get("netIncomeToCommon")),- ("Profit Margin", info.get("profitMargins")),- ("Operating Margin", info.get("operatingMargins")),- ("Return on Equity", info.get("returnOnEquity")),- ("Return on Assets", info.get("returnOnAssets")),- ("Debt to Equity", info.get("debtToEquity")),- ("Current Ratio", info.get("currentRatio")),- ("Book Value", info.get("bookValue")),- ("Free Cash Flow", info.get("freeCashflow")),- ]-- lines = [f"{label}: {v}" for label, v in fields if v is not None]-- # yfinance returns a stub dict (e.g. {"trailingPegRatio": None}) for- # unknown symbols, so `info` is truthy but every field is empty. Treat- # "no usable fields" as no data rather than emitting a bare header the- # agent might fabricate around.- if not lines:- raise NoMarketDataError(ticker, canonical, "no fundamental fields returned")-- header = f"# Company Fundamentals for {canonical}\n\n"-- return header + "\n".join(lines)-- except VendorError:- raise- except Exception as e:- raise NoMarketDataError(ticker, canonical, f"fundamentals unavailable: {e}") from e+ info = yf_retry(lambda: yf.Ticker(canonical).info)+ if not info:+ raise_for_empty(ticker, canonical, "fundamentals")++ fields = [+ ("Name", info.get("longName")),+ ("Sector", info.get("sector")),+ ("Industry", info.get("industry")),+ ("Market Cap", info.get("marketCap")),+ ("PE Ratio (TTM)", info.get("trailingPE")),+ ("Forward PE", info.get("forwardPE")),+ ("PEG Ratio", info.get("pegRatio")),+ ("Price to Book", info.get("priceToBook")),+ ("EPS (TTM)", info.get("trailingEps")),+ ("Forward EPS", info.get("forwardEps")),+ ("Dividend Yield", info.get("dividendYield")),+ ("Beta", info.get("beta")),+ ("52 Week High", info.get("fiftyTwoWeekHigh")),+ ("52 Week Low", info.get("fiftyTwoWeekLow")),+ ("50 Day Average", info.get("fiftyDayAverage")),+ ("200 Day Average", info.get("twoHundredDayAverage")),+ ("Revenue (TTM)", info.get("totalRevenue")),+ ("Gross Profit", info.get("grossProfits")),+ ("EBITDA", info.get("ebitda")),+ ("Net Income", info.get("netIncomeToCommon")),+ ("Profit Margin", info.get("profitMargins")),+ ("Operating Margin", info.get("operatingMargins")),+ ("Return on Equity", info.get("returnOnEquity")),+ ("Return on Assets", info.get("returnOnAssets")),+ ("Debt to Equity", info.get("debtToEquity")),+ ("Current Ratio", info.get("currentRatio")),+ ("Book Value", info.get("bookValue")),+ ("Free Cash Flow", info.get("freeCashflow")),+ ]++ lines = [f"{label}: {v}" for label, v in fields if v is not None]++ # yfinance returns a stub dict (e.g. {"trailingPegRatio": None}) for+ # unknown symbols, so `info` is truthy but every field is empty. Treat+ # "no usable fields" as no data rather than emitting a bare header the+ # agent might fabricate around.+ if not lines:+ raise_for_empty(ticker, canonical, "fundamental fields")++ return f"# Company Fundamentals for {canonical}\n\n" + "\n".join(lines)# This vendor dates a statement by the period it covers, not by the day it was@@ -103,17 +94,11 @@ def _statement(ticker, freq, as_of_date, title, quarterly_attr, annual_attr) ->"""One financial statement as CSV, cut at ``as_of_date`` by period end."""canonical = normalize_symbol(ticker)what = title.lower()- try:- ticker_obj = yf.Ticker(canonical)- attr = quarterly_attr if freq.lower() == "quarterly" else annual_attr- data = filter_financials_by_date(yf_retry(lambda: getattr(ticker_obj, attr)), as_of_date)- if data.empty:- raise_for_empty(ticker, canonical, f"{what} data")- return f"# {title} data for {canonical} ({freq})\n" + _PERIOD_END_VINTAGE + data.to_csv()- except VendorError:- raise- except Exception as e:- raise NoMarketDataError(ticker, canonical, f"{what} unavailable: {e}") from e+ attr = quarterly_attr if freq.lower() == "quarterly" else annual_attr+ data = filter_financials_by_date(yf_retry(lambda: getattr(yf.Ticker(canonical), attr)), as_of_date)+ if data.empty:+ raise_for_empty(ticker, canonical, f"{what} data")+ return f"# {title} data for {canonical} ({freq})\n" + _PERIOD_END_VINTAGE + data.to_csv()def get_balance_sheet(@@ -160,42 +145,31 @@ def get_insider_transactions():"""Get insider transactions data from yfinance."""canonical = normalize_symbol(ticker)- try:- ticker_obj = yf.Ticker(canonical)- data = yf_retry(lambda: ticker_obj.insider_transactions)-- # Empty is normal here (many valid symbols have no insider filings),- # so report it plainly rather than treating the symbol as invalid.- if data is None or data.empty:- if not vendor_reachable(YAHOO_HOST):- raise VendorRateLimitError("Yahoo Finance is unreachable; insider filings were not retrieved")- return f"No insider transactions reported for symbol '{canonical}'"-- if as_of_date:- traded = data["Start Date"]- kept = data[traded <= pd.Timestamp(as_of_date)]- if kept.empty:- return (- f"<insider transactions unavailable for {canonical} as of {as_of_date}: "- "Yahoo serves recent transactions only>"- )- data = kept-- return f"# Insider Transactions data for {canonical}\n" + _TRANSACTION_DATE_VINTAGE + data.to_csv()-- except VendorError:- raise- except Exception as e:- raise NoMarketDataError(ticker, canonical, f"insider transactions unavailable: {e}") from e+ data = yf_retry(lambda: yf.Ticker(canonical).insider_transactions)++ # Empty is normal here (many valid symbols have no insider filings),+ # so report it plainly rather than treating the symbol as invalid.+ if data is None or data.empty:+ if not vendor_reachable(YAHOO_HOST):+ raise VendorRateLimitError("Yahoo Finance is unreachable; insider filings were not retrieved")+ return f"No insider transactions reported for symbol '{canonical}'"++ if as_of_date:+ traded = data["Start Date"]+ kept = data[traded <= pd.Timestamp(as_of_date)]+ if kept.empty:+ return (+ f"<insider transactions unavailable for {canonical} as of {as_of_date}: "+ "Yahoo serves recent transactions only>"+ )+ data = kept++ return f"# Insider Transactions data for {canonical}\n" + _TRANSACTION_DATE_VINTAGE + data.to_csv()def get_company_profile(ticker: str) -> dict:"""Yahoo's current profile for ``ticker``: name, sector, industry and the like."""- canonical = normalize_symbol(ticker)- try:- return yf_retry(lambda: yf.Ticker(canonical).info) or {}- except Exception as e:- raise NoMarketDataError(ticker, canonical, f"profile unavailable: {e}") from e+ return yf_retry(lambda: yf.Ticker(normalize_symbol(ticker)).info) or {}def filter_financials_by_date(data: pd.DataFrame, as_of_date: str) -> pd.DataFrame:diff --git a/tradingagents/dataflows/vendors/yahoo/market.py b/tradingagents/dataflows/vendors/yahoo/market.pyindex 6965e54d2..cc65ef2d5 100644--- a/tradingagents/dataflows/vendors/yahoo/market.py+++ b/tradingagents/dataflows/vendors/yahoo/market.py@@ -273,10 +273,7 @@ def get_stockstats_indicator(def get_closes(symbol: str, start_date: str, end_date: str) -> pd.Series:"""Daily closes from ``start_date`` up to, not including, ``end_date``."""canonical = normalize_symbol(symbol)- try:- history = yf_retry(lambda: yf.Ticker(canonical).history(start=start_date, end=end_date))- except Exception as e:- raise NoMarketDataError(symbol, canonical, f"prices unavailable: {e}") from e+ history = yf_retry(lambda: yf.Ticker(canonical).history(start=start_date, end=end_date))return history["Close"] if "Close" in history else pd.Series(dtype=float)diff --git a/tradingagents/dataflows/vendors/yahoo/news.py b/tradingagents/dataflows/vendors/yahoo/news.pyindex 04fe18f8a..1a17ddf67 100644--- a/tradingagents/dataflows/vendors/yahoo/news.py+++ b/tradingagents/dataflows/vendors/yahoo/news.py@@ -8,7 +8,6 @@ from dateutil.relativedelta import relativedeltafrom tradingagents.dataflows.config import get_configfrom tradingagents.dataflows.date_window import coverage_gap, in_window-from tradingagents.dataflows.errors import NoMarketDataError, VendorErrorfrom tradingagents.dataflows.symbols import normalize_symbolfrom tradingagents.dataflows.vendors.yahoo.ohlcv import yf_retry@@ -80,44 +79,38 @@ def get_news_yfinance(# returns no news. Keep the user's ticker in the report header.canonical = normalize_symbol(ticker)resolved = "" if canonical == ticker else f" (resolved to {canonical})"- try:- stock = yf.Ticker(canonical)- news = yf_retry(lambda: stock.get_news(count=article_limit)) or []+ news = yf_retry(lambda: yf.Ticker(canonical).get_news(count=article_limit)) or []- start_dt = datetime.strptime(start_date, "%Y-%m-%d")- end_dt = datetime.strptime(end_date, "%Y-%m-%d")+ start_dt = datetime.strptime(start_date, "%Y-%m-%d")+ end_dt = datetime.strptime(end_date, "%Y-%m-%d")- news_str = ""- filtered_count = 0+ news_str = ""+ filtered_count = 0- for article in news:- data = _extract_article_data(article)+ for article in news:+ data = _extract_article_data(article)- # Keep only articles within the requested window (look-ahead safe).- if not in_window(data["pub_date"], start_dt, end_dt):- continue+ # Keep only articles within the requested window (look-ahead safe).+ if not in_window(data["pub_date"], start_dt, end_dt):+ continue- news_str += f"### {data['title']} (source: {data['publisher']})\n"- if data["summary"]:- news_str += f"{data['summary']}\n"- if data["link"]:- news_str += f"Link: {data['link']}\n"- news_str += "\n"- filtered_count += 1+ news_str += f"### {data['title']} (source: {data['publisher']})\n"+ if data["summary"]:+ news_str += f"{data['summary']}\n"+ if data["link"]:+ news_str += f"Link: {data['link']}\n"+ news_str += "\n"+ filtered_count += 1- if filtered_count == 0:- gap = coverage_gap(- (_extract_article_data(a)["pub_date"] for a in news),- start_date, end_date, "Yahoo Finance news", f"news for {ticker}{resolved}",- )- return gap or f"No news found for {ticker}{resolved} between {start_date} and {end_date}"+ if filtered_count == 0:+ gap = coverage_gap(+ (_extract_article_data(a)["pub_date"] for a in news),+ start_date, end_date, "Yahoo Finance news", f"news for {ticker}{resolved}",+ )+ return gap or f"No news found for {ticker}{resolved} between {start_date} and {end_date}"- return f"## {ticker}{resolved} News, from {start_date} to {end_date}:\n\n{news_str}"+ return f"## {ticker}{resolved} News, from {start_date} to {end_date}:\n\n{news_str}"- except VendorError:- raise- except Exception as e:- raise NoMarketDataError(ticker, ticker, f"news unavailable: {e}") from edef get_global_news_yfinance(@@ -152,49 +145,43 @@ def get_global_news_yfinance(in_window_news = []seen_titles = set()- try:- for query in search_queries:- search = yf_retry(lambda q=query: yf.Search(- query=q,- news_count=limit,- enable_fuzzy_query=True,- ))-- for article in search.news or []:- # Window first: the limit counts what the run may read, so an- # out-of-window article must not spend the budget or cut the- # remaining searches short (#1356). Flat articles are filtered- # on the same rule, so none can leak future news (#1007).- data = _extract_article_data(article)- if not in_window(data["pub_date"], start_dt, curr_dt):- continue- if data["title"] and data["title"] not in seen_titles:- seen_titles.add(data["title"])- in_window_news.append(data)-- if len(in_window_news) >= limit:- break-- news_str = ""- for data in in_window_news[:limit]:- news_str += f"### {data['title']} (source: {data['publisher']})\n"- if data["summary"]:- news_str += f"{data['summary']}\n"- if data["link"]:- news_str += f"Link: {data['link']}\n"- news_str += "\n"-- # Nothing fell inside the window -> say so rather than return an- # empty-bodied report (#993).- if not news_str:- # Results merge several fuzzy searches, so their timestamps prove no- # continuous coverage; judge the window against the present only.- gap = coverage_gap((), start_date, as_of_date, "Yahoo Finance global news", "market news")- return gap or f"No global news found between {start_date} and {as_of_date}"-- return f"## Global Market News, from {start_date} to {as_of_date}:\n\n{news_str}"-- except VendorError:- raise- except Exception as e:- raise NoMarketDataError("global news", "global news", f"unavailable: {e}") from e+ for query in search_queries:+ found = yf_retry(lambda q=query: yf.Search(+ query=q,+ news_count=limit,+ enable_fuzzy_query=True,+ ).news)++ for article in found or []:+ # Window first: the limit counts what the run may read, so an+ # out-of-window article must not spend the budget or cut the+ # remaining searches short (#1356). Flat articles are filtered+ # on the same rule, so none can leak future news (#1007).+ data = _extract_article_data(article)+ if not in_window(data["pub_date"], start_dt, curr_dt):+ continue+ if data["title"] and data["title"] not in seen_titles:+ seen_titles.add(data["title"])+ in_window_news.append(data)++ if len(in_window_news) >= limit:+ break++ news_str = ""+ for data in in_window_news[:limit]:+ news_str += f"### {data['title']} (source: {data['publisher']})\n"+ if data["summary"]:+ news_str += f"{data['summary']}\n"+ if data["link"]:+ news_str += f"Link: {data['link']}\n"+ news_str += "\n"++ # Nothing fell inside the window -> say so rather than return an+ # empty-bodied report (#993).+ if not news_str:+ # Results merge several fuzzy searches, so their timestamps prove no+ # continuous coverage; judge the window against the present only.+ gap = coverage_gap((), start_date, as_of_date, "Yahoo Finance global news", "market news")+ return gap or f"No global news found between {start_date} and {as_of_date}"++ return f"## Global Market News, from {start_date} to {as_of_date}:\n\n{news_str}"diff --git a/tradingagents/dataflows/vendors/yahoo/ohlcv.py b/tradingagents/dataflows/vendors/yahoo/ohlcv.pyindex ddcd58609..b9b0a710e 100644--- a/tradingagents/dataflows/vendors/yahoo/ohlcv.py+++ b/tradingagents/dataflows/vendors/yahoo/ohlcv.py@@ -43,12 +43,13 @@ def yf_retry(func, max_retries=3, base_delay=2.0):"""Execute a yfinance call with exponential backoff on rate limits.yfinance raises YFRateLimitError on HTTP 429 responses but does not- retry them internally. This wrapper adds retry logic specifically- for rate limits. Other exceptions propagate immediately. A rate limit- that outlasts the retries is raised as VendorRateLimitError, so the- router reports a throttled vendor rather than a symbol with no data.- ``func`` should build its own Ticker: a Ticker keeps a failed ``info``- fetch as done, so asking the same one again reads an empty profile.+ retry them internally, so this wrapper retries them. A rate limit that+ outlasts the retries, or any other exception, is raised as+ VendorRateLimitError: Yahoo answers an unknown symbol with an empty+ result, so a request that raised failed in transit and says nothing about+ the symbol. ``func`` should build its own Ticker and make the request+ itself: a Ticker keeps a failed ``info`` fetch as done, so asking the same+ one again reads an empty profile."""for attempt in range(max_retries + 1):try:@@ -62,6 +63,8 @@ def yf_retry(func, max_retries=3, base_delay=2.0):raise VendorRateLimitError(f"Yahoo Finance rate limited after {max_retries} retries: {exc}") from exc+ except Exception as exc:+ raise VendorRateLimitError(f"Yahoo Finance request failed: {type(exc).__name__}") from excdef _ensure_date_column(data: pd.DataFrame) -> pd.DataFrame:@@ -249,20 +252,12 @@ def load_ohlcv(symbol: str, as_of_date: str, fill_gaps: bool = True) -> pd.DataFif data is None:# yf.download catches every error, a rate limit included, and returns# an empty frame. Ticker.history raises the rate limit, so it is retried.- try:- downloaded = yf_retry(lambda: yf.Ticker(canonical).history(- start=start_str,- end=end_str,- auto_adjust=True,- actions=False,- ))- except VendorRateLimitError:- raise- except Exception as exc:- # Any other failure is an outage or an unknown symbol, which- # raise_for_empty tells apart by whether Yahoo answers at all.- logger.warning("Yahoo Finance price request for %s failed: %s", canonical, exc)- raise_for_empty(symbol, canonical, "price rows")+ downloaded = yf_retry(lambda: yf.Ticker(canonical).history(+ start=start_str,+ end=end_str,+ auto_adjust=True,+ actions=False,+ ))downloaded = _ensure_date_column(downloaded.reset_index())# Only cache real data — never persist an empty frame.if downloaded.empty or "Close" not in downloaded.columns: