Skip to content
Python

fix(dataflows): map crypto to StockTwits/Reddit sentiment symbols

Yijia-Xiaoa102afa090da
diff --git a/tests/test_reddit_fallback.py b/tests/test_reddit_fallback.py
index 7fd98a90c..dece01ca6 100644
--- a/tests/test_reddit_fallback.py
+++ b/tests/test_reddit_fallback.py
@@ -190,3 +190,25 @@ class TestFormatterHandlesRssPosts:
assert "1234↑" in out
assert "56c" in out
assert "via RSS" not in out
+
+
+class TestCryptoSearchTerm:
+ """A crypto pair (BTC-USD) barely matches Reddit text; search the base (#1113)."""
+
+ def _captured_ticker(self, ticker):
+ seen = {}
+
+ def fake_fetch(t, sub, limit, timeout):
+ seen["ticker"] = t
+ return []
+
+ with patch.object(reddit, "_fetch_subreddit", side_effect=fake_fetch):
+ reddit.fetch_reddit_posts(ticker, subreddits=("stocks",), inter_request_delay=0)
+ return seen["ticker"]
+
+ def test_crypto_pair_searches_base(self):
+ assert self._captured_ticker("BTC-USD") == "BTC"
+
+ def test_equity_passes_through(self):
+ assert self._captured_ticker("NVDA") == "NVDA"
diff --git a/tests/test_stocktwits_resilience.py b/tests/test_stocktwits_resilience.py
index 145e2267d..8e963f5c5 100644
--- a/tests/test_stocktwits_resilience.py
+++ b/tests/test_stocktwits_resilience.py
@@ -1,5 +1,9 @@
-"""StockTwits fetch degrades (never raises) on transport errors, including the
-http.client chunked-transfer exceptions that are not OSErrors (#1024)."""
+"""StockTwits fetch: transport-error resilience (#1024) and crypto symbol
+mapping (#1113).
+
+StockTwits lists crypto under ``<BASE>.X`` (Yahoo's ``BTC-USD`` 404s), and any
+transport error must degrade to a placeholder rather than raise.
+"""
from __future__ import annotations
@@ -26,7 +30,7 @@ def _raise(exc):
@pytest.mark.unit
-class StockTwitsResilienceTests:
+class TestStockTwitsResilience:
@pytest.mark.parametrize(
"exc",
[
@@ -40,3 +44,34 @@ class StockTwitsResilienceTests:
out = stocktwits.fetch_stocktwits_messages("NVDA")
assert "unavailable" in out.lower()
assert out.startswith("<stocktwits unavailable")
+
+
+class TestStockTwitsCryptoSymbols:
+ @pytest.mark.parametrize(
+ ("ticker", "expected"),
+ [
+ ("BTC-USD", "BTC.X"),
+ ("eth-usd", "ETH.X"),
+ ("SOL-USD", "SOL.X"),
+ ("BTCUSD", "BTC.X"), # undashed broker form
+ ("BTC-USDT", "BTC.X"), # stablecoin quote
+ ("AMD", "AMD"),
+ ("BRK-B", "BRK-B"), # dashed class share: untouched
+ ("GOLD", "GOLD"), # real equity (aliases elsewhere): untouched here
+ ("XYZ-USD", "XYZ-USD"), # unknown base: not treated as crypto
+ ],
+ )
+ def test_symbol_mapping(self, ticker, expected):
+ assert stocktwits._stocktwits_symbol(ticker) == expected
+
+ def test_crypto_pair_requests_dot_x_endpoint(self):
+ seen = {}
+
+ def fake_urlopen(req, timeout=None):
+ seen["url"] = req.full_url
+ raise TimeoutError("stop after capturing the URL")
+
+ with patch.object(stocktwits, "urlopen", side_effect=fake_urlopen):
+ stocktwits.fetch_stocktwits_messages("BTC-USD")
+ assert "/symbol/BTC.X.json" in seen["url"]
diff --git a/tests/test_symbol_utils.py b/tests/test_symbol_utils.py
index 774c0f9bf..9db3894ff 100644
--- a/tests/test_symbol_utils.py
+++ b/tests/test_symbol_utils.py
@@ -6,6 +6,7 @@ import pytest
from tradingagents.dataflows.symbol_utils import (
NoMarketDataError,
+ crypto_base,
is_yahoo_safe,
normalize_symbol,
)
@@ -77,5 +78,25 @@ class TestIsYahooSafe(unittest.TestCase):
self.assertFalse(is_yahoo_safe(sym))
+class TestCryptoBase(unittest.TestCase):
+ def test_resolves_known_crypto_forms(self):
+ for raw in ("BTC-USD", "BTCUSD", "btc-usdt", "BTC-USDC", "BTCUSD+"):
+ self.assertEqual(crypto_base(raw), "BTC")
+ self.assertEqual(crypto_base("ETH-USD"), "ETH")
+ self.assertEqual(crypto_base("sol-usd"), "SOL")
+
+ def test_non_crypto_returns_none(self):
+ # Plain equities, class shares, and real tickers that alias elsewhere
+ # (GOLD -> gold future on the Yahoo path) must NOT read as crypto.
+ for raw in ("AAPL", "BRK-B", "GOLD", "XYZ-USD", "EURUSD", "", None):
+ self.assertIsNone(crypto_base(raw))
+
+ def test_agrees_with_normalize_symbol(self):
+ # crypto_base is the shared primitive behind the -USD normalization.
+ self.assertEqual(normalize_symbol("BTCUSD"), "BTC-USD")
+ self.assertEqual(crypto_base("BTCUSD"), "BTC")
+
+
if __name__ == "__main__":
unittest.main()
diff --git a/tradingagents/dataflows/reddit.py b/tradingagents/dataflows/reddit.py
index bfb87700b..edb4aedfc 100644
--- a/tradingagents/dataflows/reddit.py
+++ b/tradingagents/dataflows/reddit.py
@@ -30,6 +30,8 @@ from urllib.error import HTTPError
from urllib.parse import urlencode
from urllib.request import Request, urlopen
+from .symbol_utils import crypto_base
+
logger = logging.getLogger(__name__)
_API = "https://www.reddit.com/r/{sub}/search.json?{qs}"
@@ -200,6 +202,9 @@ def fetch_reddit_posts(
stay under Reddit's public per-IP rate limit; combined with the RSS-first
path it makes 429s rare even when several analyses run back-to-back.
"""
+ # Crypto reaches us as a Yahoo pair (BTC-USD); search Reddit for the base
+ # ("BTC") so the query actually matches discussion instead of near-nothing.
+ ticker = crypto_base(ticker) or ticker
blocks = []
total_posts = 0
for i, sub in enumerate(subreddits):
diff --git a/tradingagents/dataflows/stocktwits.py b/tradingagents/dataflows/stocktwits.py
index 1594de1d3..1c2fd02bb 100644
--- a/tradingagents/dataflows/stocktwits.py
+++ b/tradingagents/dataflows/stocktwits.py
@@ -19,12 +19,25 @@ import json
import logging
from urllib.request import Request, urlopen
+from .symbol_utils import crypto_base
+
logger = logging.getLogger(__name__)
_API = "https://api.stocktwits.com/api/2/streams/symbol/{ticker}.json"
_UA = "tradingagents/0.2 (+https://github.com/TauricResearch/TradingAgents)"
+def _stocktwits_symbol(ticker: str) -> str:
+ """Map a crypto pair to StockTwits' ``<BASE>.X`` convention.
+
+ StockTwits lists crypto as ``BTC.X`` (Yahoo's ``BTC-USD`` form 404s), so any
+ crypto symbol resolves to its base plus ``.X``; other symbols pass through
+ upper-cased.
+ """
+ base = crypto_base(ticker)
+ return f"{base}.X" if base else ticker.strip().upper()
+
+
def fetch_stocktwits_messages(ticker: str, limit: int = 30, timeout: float = 10.0) -> str:
"""Fetch recent StockTwits messages for ``ticker`` and return them as a
formatted plaintext block ready for prompt injection.
@@ -33,7 +46,7 @@ def fetch_stocktwits_messages(ticker: str, limit: int = 30, timeout: float = 10.
symbol has no messages, or the response shape is unexpected — the
caller never has to special-case None or exceptions.
"""
- url = _API.format(ticker=ticker.upper())
+ url = _API.format(ticker=_stocktwits_symbol(ticker))
req = Request(url, headers={"User-Agent": _UA, "Accept": "application/json"})
try:
with urlopen(req, timeout=timeout) as resp:
diff --git a/tradingagents/dataflows/symbol_utils.py b/tradingagents/dataflows/symbol_utils.py
index af46a1f02..f91b4e149 100644
--- a/tradingagents/dataflows/symbol_utils.py
+++ b/tradingagents/dataflows/symbol_utils.py
@@ -80,22 +80,27 @@ _YAHOO_SAFE = re.compile(r"^[A-Za-z0-9._\-\^=]+$")
_CRYPTO_QUOTES = ("USDT", "USDC", "USD")
-def _normalize_crypto(s: str) -> str | None:
- """Return ``<BASE>-USD`` if ``s`` is a known crypto quoted in USD/USDT/USDC.
-
- Accepts dashed or undashed forms: ``BTCUSD``, ``BTCUSDT``, ``BTC-USDT``,
- ``BTC-USDC`` all resolve to ``BTC-USD``. Returns None otherwise.
+def crypto_base(raw: str) -> str | None:
+ """Return the crypto base (e.g. ``BTC``) for a known USD/USDT/USDC-quoted
+ crypto symbol in any form the pipeline may hold — ``BTC-USD``, ``BTCUSD``,
+ ``BTC-USDT`` — or None for non-crypto symbols. Purely syntactic.
"""
- compact = s.replace("-", "")
+ if not isinstance(raw, str):
+ return None
+ compact = raw.strip().upper().rstrip("+").replace("-", "")
for quote in _CRYPTO_QUOTES:
if compact.endswith(quote):
base = compact[: -len(quote)]
- if base in _CRYPTO_BASES:
- return f"{base}-USD"
- break
+ return base if base in _CRYPTO_BASES else None
return None
+def _normalize_crypto(s: str) -> str | None:
+ """Return ``<BASE>-USD`` for a known USD/USDT/USDC-quoted crypto, else None."""
+ base = crypto_base(s)
+ return f"{base}-USD" if base else None
+
+
def normalize_symbol(raw: str) -> str:
"""Map a user/broker symbol to its canonical Yahoo Finance symbol.