Skip to content
Python

fix(dataflows): trim global news to the window before the limit (#1356)

Yijia-Xiao8d6441643681
diff --git a/tests/test_news_lookahead.py b/tests/test_news_lookahead.py
index 7651cff4c..2d7106415 100644
--- a/tests/test_news_lookahead.py
+++ b/tests/test_news_lookahead.py
@@ -213,3 +213,47 @@ def test_coverage_gap_future_window_is_unavailable():
today = datetime.now(timezone.utc).date()
out = coverage_gap([], str(today), str(today + timedelta(days=3)), "Feed", "items")
assert out is not None and "past today" in out
+
+
+def test_out_of_window_articles_do_not_consume_the_article_budget(monkeypatch):
+ """The limit counts articles the run may see, not candidates fetched (#1356).
+
+ Out-of-window items were counted first, so they filled the budget, stopped
+ the remaining searches, and the in-window news was reported as absent.
+ """
+ stale = [{"title": f"OLD {i}", "publisher": "P", "link": "l",
+ "providerPublishTime": _epoch("2025-01-01")} for i in range(2)]
+ wanted = {"title": "IN WINDOW", "publisher": "P", "link": "l",
+ "providerPublishTime": _epoch("2025-05-08")}
+ pages = [stale, [wanted]]
+
+ class FakeSearch:
+ def __init__(self, *a, **k):
+ self.news = pages.pop(0) if pages else []
+
+ monkeypatch.setattr(ynews.yf, "Search", FakeSearch)
+ monkeypatch.setattr(ynews, "get_config", lambda: {
+ "global_news_lookback_days": 7, "global_news_article_limit": 2,
+ "global_news_queries": ["markets", "economy"],
+ })
+
+ out = ynews.get_global_news_yfinance("2025-05-09")
+
+ assert "IN WINDOW" in out
+ assert "OLD 0" not in out
+
+
+def test_the_article_limit_still_caps_what_is_returned(monkeypatch):
+ articles = [{"title": f"NEWS {i}", "publisher": "P", "link": "l",
+ "providerPublishTime": _epoch("2025-05-08")} for i in range(5)]
+
+ class FakeSearch:
+ def __init__(self, *a, **k):
+ self.news = articles
+
+ monkeypatch.setattr(ynews.yf, "Search", FakeSearch)
+ out = ynews.get_global_news_yfinance("2025-05-09", look_back_days=7, limit=3)
+
+ assert out.count("### ") == 3
diff --git a/tradingagents/dataflows/yfinance_news.py b/tradingagents/dataflows/yfinance_news.py
index e330a7af2..033afcd8c 100644
--- a/tradingagents/dataflows/yfinance_news.py
+++ b/tradingagents/dataflows/yfinance_news.py
@@ -146,7 +146,11 @@ def get_global_news_yfinance(
limit = config["global_news_article_limit"]
search_queries = config["global_news_queries"]
- all_news = []
+ curr_dt = datetime.strptime(curr_date, "%Y-%m-%d")
+ start_dt = curr_dt - relativedelta(days=look_back_days)
+ start_date = start_dt.strftime("%Y-%m-%d")
+
+ in_window_news = []
seen_titles = set()
try:
@@ -157,47 +161,33 @@ def get_global_news_yfinance(
enable_fuzzy_query=True,
))
- if search.news:
- for article in search.news:
- # Handle both flat and nested structures
- if "content" in article:
- data = _extract_article_data(article)
- title = data["title"]
- else:
- title = article.get("title", "")
-
- # Deduplicate by title
- if title and title not in seen_titles:
- seen_titles.add(title)
- all_news.append(article)
-
- if len(all_news) >= limit:
+ for article in search.news or []:
+ # Window first: the limit counts what the run may read, so an
+ # out-of-window article must not spend the budget or cut the
+ # remaining searches short (#1356). Flat articles are filtered
+ # on the same rule, so none can leak future news (#1007).
+ data = _extract_article_data(article)
+ if not in_window(data["pub_date"], start_dt, curr_dt):
+ continue
+ if data["title"] and data["title"] not in seen_titles:
+ seen_titles.add(data["title"])
+ in_window_news.append(data)
+
+ if len(in_window_news) >= limit:
break
- # Calculate date range
- curr_dt = datetime.strptime(curr_date, "%Y-%m-%d")
- start_dt = curr_dt - relativedelta(days=look_back_days)
- start_date = start_dt.strftime("%Y-%m-%d")
-
news_str = ""
- kept = 0
- for article in all_news[:limit]:
- # Extract uniformly (flat + nested) and apply the same look-ahead-safe
- # window filter, so flat articles can't leak future news (#1007).
- data = _extract_article_data(article)
- if not in_window(data["pub_date"], start_dt, curr_dt):
- continue
+ for data in in_window_news[:limit]:
news_str += f"### {data['title']} (source: {data['publisher']})\n"
if data["summary"]:
news_str += f"{data['summary']}\n"
if data["link"]:
news_str += f"Link: {data['link']}\n"
news_str += "\n"
- kept += 1
- # All candidates fell outside the window -> say so rather than return an
+ # Nothing fell inside the window -> say so rather than return an
# empty-bodied report (#993).
- if kept == 0:
+ if not news_str:
# Results merge several fuzzy searches, so their timestamps prove no
# continuous coverage; judge the window against the present only.
gap = coverage_gap((), start_date, curr_date, "Yahoo Finance global news", "market news")