Python
fix(dataflows): trim global news to the window before the limit (#1356)
Yijia-Xiao8d6441643681
diff --git a/tests/test_news_lookahead.py b/tests/test_news_lookahead.pyindex 7651cff4c..2d7106415 100644--- a/tests/test_news_lookahead.py+++ b/tests/test_news_lookahead.py@@ -213,3 +213,47 @@ def test_coverage_gap_future_window_is_unavailable():today = datetime.now(timezone.utc).date()out = coverage_gap([], str(today), str(today + timedelta(days=3)), "Feed", "items")assert out is not None and "past today" in out+++def test_out_of_window_articles_do_not_consume_the_article_budget(monkeypatch):+ """The limit counts articles the run may see, not candidates fetched (#1356).++ Out-of-window items were counted first, so they filled the budget, stopped+ the remaining searches, and the in-window news was reported as absent.+ """+ stale = [{"title": f"OLD {i}", "publisher": "P", "link": "l",+ "providerPublishTime": _epoch("2025-01-01")} for i in range(2)]+ wanted = {"title": "IN WINDOW", "publisher": "P", "link": "l",+ "providerPublishTime": _epoch("2025-05-08")}+ pages = [stale, [wanted]]++ class FakeSearch:+ def __init__(self, *a, **k):+ self.news = pages.pop(0) if pages else []++ monkeypatch.setattr(ynews.yf, "Search", FakeSearch)+ monkeypatch.setattr(ynews, "get_config", lambda: {+ "global_news_lookback_days": 7, "global_news_article_limit": 2,+ "global_news_queries": ["markets", "economy"],+ })++ out = ynews.get_global_news_yfinance("2025-05-09")++ assert "IN WINDOW" in out+ assert "OLD 0" not in out+++def test_the_article_limit_still_caps_what_is_returned(monkeypatch):+ articles = [{"title": f"NEWS {i}", "publisher": "P", "link": "l",+ "providerPublishTime": _epoch("2025-05-08")} for i in range(5)]++ class FakeSearch:+ def __init__(self, *a, **k):+ self.news = articles++ monkeypatch.setattr(ynews.yf, "Search", FakeSearch)+ out = ynews.get_global_news_yfinance("2025-05-09", look_back_days=7, limit=3)++ assert out.count("### ") == 3diff --git a/tradingagents/dataflows/yfinance_news.py b/tradingagents/dataflows/yfinance_news.pyindex e330a7af2..033afcd8c 100644--- a/tradingagents/dataflows/yfinance_news.py+++ b/tradingagents/dataflows/yfinance_news.py@@ -146,7 +146,11 @@ def get_global_news_yfinance(limit = config["global_news_article_limit"]search_queries = config["global_news_queries"]- all_news = []+ curr_dt = datetime.strptime(curr_date, "%Y-%m-%d")+ start_dt = curr_dt - relativedelta(days=look_back_days)+ start_date = start_dt.strftime("%Y-%m-%d")++ in_window_news = []seen_titles = set()try:@@ -157,47 +161,33 @@ def get_global_news_yfinance(enable_fuzzy_query=True,))- if search.news:- for article in search.news:- # Handle both flat and nested structures- if "content" in article:- data = _extract_article_data(article)- title = data["title"]- else:- title = article.get("title", "")-- # Deduplicate by title- if title and title not in seen_titles:- seen_titles.add(title)- all_news.append(article)-- if len(all_news) >= limit:+ for article in search.news or []:+ # Window first: the limit counts what the run may read, so an+ # out-of-window article must not spend the budget or cut the+ # remaining searches short (#1356). Flat articles are filtered+ # on the same rule, so none can leak future news (#1007).+ data = _extract_article_data(article)+ if not in_window(data["pub_date"], start_dt, curr_dt):+ continue+ if data["title"] and data["title"] not in seen_titles:+ seen_titles.add(data["title"])+ in_window_news.append(data)++ if len(in_window_news) >= limit:break- # Calculate date range- curr_dt = datetime.strptime(curr_date, "%Y-%m-%d")- start_dt = curr_dt - relativedelta(days=look_back_days)- start_date = start_dt.strftime("%Y-%m-%d")-news_str = ""- kept = 0- for article in all_news[:limit]:- # Extract uniformly (flat + nested) and apply the same look-ahead-safe- # window filter, so flat articles can't leak future news (#1007).- data = _extract_article_data(article)- if not in_window(data["pub_date"], start_dt, curr_dt):- continue+ for data in in_window_news[:limit]:news_str += f"### {data['title']} (source: {data['publisher']})\n"if data["summary"]:news_str += f"{data['summary']}\n"if data["link"]:news_str += f"Link: {data['link']}\n"news_str += "\n"- kept += 1- # All candidates fell outside the window -> say so rather than return an+ # Nothing fell inside the window -> say so rather than return an# empty-bodied report (#993).- if kept == 0:+ if not news_str:# Results merge several fuzzy searches, so their timestamps prove no# continuous coverage; judge the window against the present only.gap = coverage_gap((), start_date, curr_date, "Yahoo Finance global news", "market news")