"Vibe-Trading: Your Personal Trading Agent"
Python ai-agentalgorithmic-tradingbacktestingfintechllmmcpmulti-agentpythonquantitative-financetrading
"""Read-only symbol-search tool: resolve a name/ticker to symbols + market.
Backed by the selected Binance connector for exact crypto pairs plus three
frozen, IP-throttled public-API clients so the agent never hits a provider
un-throttled and never re-implements transport plumbing:
* The active Binance profile resolves exact spot-pair spellings against its
exchange market catalog. It does not guess asset names from prose.
* :mod:`backtest.loaders.eastmoney_client` — Eastmoney's free suggest endpoint
matches Chinese/English names and tickers across A-shares (.SH/.SZ/.BJ),
Hong Kong (.HK) and U.S. (.US) listings, each carrying a fully-qualified
``secid`` already in ``<market>.<code>`` form.
* :mod:`backtest.loaders.yahoo_client` — Yahoo's v1 search endpoint matches
global tickers/company names (US, HK, Canada, crypto, indices, FX, ...).
* :mod:`backtest.loaders.sec_edgar_client` — the SEC company-tickers table
enriches a resolved U.S. equity ticker with its zero-padded CIK.
The tool fans out across these sources, normalizes every hit into one compact
candidate row in the project's symbol convention, de-duplicates by symbol, and
caps the payload. A single failing source never aborts the others; its error is
recorded under ``sources`` and the surviving candidates still return.
"""
from __future__ import annotations
import json
import logging
import re
from typing import Any, Dict, List, Optional
from backtest.loaders import eastmoney_client, sec_edgar_client, yahoo_client
from src.agent.tools import BaseTool
from src.market_data import FIAT_CODES, canonical_fx_pair
# Back-compat alias: the search tool's historical name for the shared
# fiat-pair canonicalizer (search, fetch and grounding share one definition).
_canonical_fx_pair = canonical_fx_pair
logger = logging.getLogger(__name__)
# Eastmoney's free, no-auth suggest endpoint (the same one the quote site calls)
# returns multi-market candidates under ``QuotationCodeTable.Data`` with a
# ready-made ``QuoteID`` secid. Requests route through the frozen, throttled
# Eastmoney client; this is just the documented endpoint URL + query shape.
_EASTMONEY_SUGGEST_URL = "https://searchapi.eastmoney.com/api/suggest/get"
# Canadian equity suffixes (TSX ``.TO`` / TSX Venture ``.V``). Eastmoney has NO
# Canada coverage: querying it with a Canadian ticker returns a non-JSON body
# (``Expecting value: line 1 column 1 (char 0)``) instead of a clean empty
# result. We fail fast — skip the endpoint entirely — for these queries, and
# drop US OTC aliases (e.g. ``BYAGF.US`` for ``BYN.V``) so the grounding
# ledger never sees one company under two venues (identity_conflict).
#
# The pattern matches a LEADING Canadian ticker, optionally followed by free
# text — the model commonly searches "BTO.TO B2Gold" or "SGML.V Sigma Lithium
# Vancouver", not just a bare "BTO.TO". ``.TO``/``.V`` are exclusively
# Canadian suffixes, so a leading one is unambiguous and Eastmoney can never
# serve it. Bare names with no suffix (e.g. "BTO", "B2Gold BTO") carry no
# venue signal and may be legit non-Canadian lookups (A-share/HK/US), so they
# are deliberately left to the normal fan-out.
_CANADIAN_SYMBOL_RE = re.compile(r"^[A-Z0-9&.\-]+\.(?:TO|V)\b", re.IGNORECASE)
# Explicit exchange-pair spellings are not equity/name searches. Restrict the
# quote leg to assets used by the built-in crypto connectors so an equity such
# as ``BRK-B`` is never misclassified as a pair. The full set is split into
# two tiers:
#
# * Stablecoin quotes (FDUSD / USDT / USDC / BUSD / TUSD) are unambiguous -
# a ``BTC-USDT`` or ``XAUT-USDC`` cannot be confused with anything outside
# crypto. These are accepted on any alphanumeric base.
# * ``USD`` is ambiguous: a ``BTC-USD`` is a real Coinbase crypto pair, but
# ``XAU-USD`` is spot gold, ``EUR-USD`` is forex, and ``GBP-USD`` is
# currency. Restrict ``USD`` to a whitelist of well-known crypto bases
# (anything that's actually tradable on Binance/OKX spot, plus the
# stablecoin-gold tokens XAUT/PAXG that ARE crypto). Without this guard a
# bare ``XAUUSD`` query would lock onto a tokenized-gold row from
# Binance instead of the spot gold the user actually asked for.
_STABLECOIN_QUOTES = ("FDUSD", "USDT", "USDC", "BUSD", "TUSD")
_CRYPTO_QUOTE_ASSETS = _STABLECOIN_QUOTES + ("BTC", "ETH", "BNB", "USD")
# Bases that may pair with ``USD`` and still count as a crypto pair. Every
# entry here is listed on Binance spot or OKX spot; ``XAU``, ``EUR``, etc.
# are deliberately excluded.
_CRYPTO_USD_BASES = frozenset(
{
"BTC", "ETH", "BNB", "SOL", "ADA", "XRP", "DOGE", "TRX", "DOT",
"MATIC", "AVAX", "LINK", "LTC", "BCH", "ETC", "XLM", "ATOM",
"FIL", "APT", "NEAR", "ALGO", "SAND", "MANA", "AXS", "XAUT",
"PAXG",
}
)
#: Spot precious metals. Their pairs are shaped exactly like FX (three-letter
#: base, fiat quote) but the base is not a fiat code, so ``canonical_fx_pair``
#: rejects them and the crypto resolver must too (``XAU-USD`` is spot gold,
#: ``XAUT-USDT`` is the token).
_METAL_CODES = frozenset({"XAU", "XAG", "XPT", "XPD"})
def _metal_or_fx_legs(value: str) -> tuple[str, str] | None:
"""Return the ``(base, quote)`` legs of a spot metal / FX pair query.
Recognizes every spelling the tool has to reconcile — ``XAUUSD``,
``XAU/USD``, ``XAU-USD``, ``XAUUSD=X`` — so a candidate written in one
spelling can be compared with a query written in another.
Args:
value: A query string or a candidate symbol.
Returns:
The uppercased legs when the base is a metal or fiat code and the
quote is a fiat code, else ``None``.
"""
clean = str(value or "").strip().upper()
if clean.endswith("=X"):
clean = clean[:-2]
if clean.endswith(".FX"):
clean = clean[:-3]
if "-" in clean or "/" in clean:
base, _, quote = (
clean.partition("-") if "-" in clean else clean.partition("/")
)
elif len(clean) == 6 and clean.isalpha():
base, quote = clean[:3], clean[3:]
else:
return None
if base in (_METAL_CODES | FIAT_CODES) and quote in FIAT_CODES:
return base, quote
return None
_CRYPTO_PAIR_RE = re.compile(
rf"^([A-Z0-9]{{2,15}})[-/]({'|'.join(_CRYPTO_QUOTE_ASSETS)})$",
re.IGNORECASE,
)
# Eastmoney market-number -> our symbol suffix. Anything else is left unmapped
# (those candidates are skipped rather than emitted with a wrong suffix).
_EASTMONEY_SUFFIX_BY_MARKET: Dict[str, str] = {
"1": "SH", # Shanghai
"0": "SZ", # Shenzhen / Beijing share the 0 prefix on Eastmoney
"116": "HK",
"105": "US", # NASDAQ
"106": "US", # NYSE
"107": "US", # AMEX
}
# Coarse market label for the candidate row, keyed by symbol suffix. Every
# market the data layer routes has a row, so none is labelled "global";
# test_market_identity_parity derives that list from the backtest's table.
_MARKET_BY_SUFFIX: Dict[str, str] = {
"SH": "cn",
"SZ": "cn",
"BJ": "cn",
"HK": "hk",
"US": "us",
"NS": "in",
"BO": "in",
"KS": "kr",
"KQ": "kr",
"TO": "ca",
"V": "ca",
"BA": "ar",
"SA": "br",
"L": "uk",
"VN": "vn",
}
# Quote types Yahoo lists under a plain US ticker (``QQQ``, ``SPY``, ``VFIAX``).
# The data layer already treats a bare ticker as ``us_equity``; only the
# resolver used to disagree, which is what #2084 reported.
_US_LISTED_QUOTE_TYPES = frozenset({"EQUITY", "ETF", "MUTUALFUND"})
# Yahoo exchange codes for US venues. Used only to keep a bare symbol that
# belongs to a non-US venue on its native form; an empty exchange is treated as
# US because Yahoo suffixes every non-US listing it lists.
_US_YAHOO_EXCHANGES = frozenset(
{
"NMS", # Nasdaq Global Select
"NGM", # Nasdaq Global Market
"NCM", # Nasdaq Capital Market
"NAS", # Nasdaq (unspecified tier)
"NYQ", # NYSE
"NYE", # NYSE (older payload spelling)
"PCX", # NYSE Arca (most ETFs)
"ASE", # NYSE American
"BTS", # Cboe BZX
"OQX", # OTCQX
"OQB", # OTCQB
"OBB", # OTC Bulletin Board
"PNK", # Pink Sheets
"OTC", # OTC (unspecified tier)
}
)
# Hard caps so a broad query cannot bloat the envelope.
_MAX_LIMIT = 25
_DEFAULT_LIMIT = 10
# Per-source fan-out ceiling before de-dup/merge keeps each provider bounded.
_PER_SOURCE_CAP = 25
# Sentinel for "no U.S. candidate, SEC was not consulted" so the caller can omit
# the ``sec_edgar`` source entry entirely.
_NO_US = "__no_us__"
# A source that cannot serve a given query shape reports this prefix instead of
# an error string. The distinction is load-bearing, not cosmetic: the grounding
# ledger concludes "this entity does not exist" only when every source that
# could answer did answer, so a source recorded as failed turns a legitimate
# "not listed" into a blocking ``invalidated`` identity. Both skip reasons below
# share this one prefix — two spellings for one concept is exactly how that
# cross-module contract breaks silently.
_SKIPPED = "skipped: "
class SymbolSearchTool(BaseTool):
"""Resolve a company name or ticker fragment to candidate symbols."""
name = "search_symbol"
description = (
"Resolve a company name or ticker fragment to candidate trading symbols "
"with their market, in the project's symbol convention (A-shares "
"600519.SH, Hong Kong 00700.HK, U.S. AAPL.US, Canada TD.TO/PNG.V, plus "
"crypto/index/FX from "
"Yahoo). Exact crypto pairs are checked against the active Binance "
"profile; other queries search Eastmoney (China/HK/US names and tickers) and Yahoo "
"(global) and, for U.S. equities, attaches the SEC CIK. Use this to turn "
"an ambiguous name into a concrete symbol before calling get_market_data "
'or get_sec_filings. Example: search_symbol(query="apple", limit=5).'
)
parameters = {
"type": "object",
"properties": {
"query": {
"type": "string",
"description": (
"Free-text company name or ticker fragment to resolve, e.g. "
"'apple', '贵州茅台', '茅台', 'AAPL', '00700'. Chinese and "
"English both accepted."
),
},
"limit": {
"type": "integer",
"description": (
f"Maximum number of merged candidates to return "
f"(1-{_MAX_LIMIT}). Defaults to {_DEFAULT_LIMIT}."
),
"default": _DEFAULT_LIMIT,
},
},
"required": ["query"],
}
repeatable = True
def execute(self, **kwargs: Any) -> str:
"""Fan out across providers and return a merged candidate envelope.
Args:
**kwargs: ``query`` (str, required free-text name/ticker) and
``limit`` (int, optional; clamped to ``1.._MAX_LIMIT``).
Returns:
A JSON envelope string. On success:
``{"ok": true, "market": "multi", "source": "symbol_search",
"data": {"query": str, "count": int, "candidates": [...],
"sources": {<name>: "ok"|<error>}}}``. On failure (only when the
query itself is invalid):
``{"ok": false, "error": str}``.
"""
query = str(kwargs.get("query") or "").strip()
if not query:
return _error("'query' is required and must be a non-empty string")
limit = _clamp_limit(kwargs.get("limit", _DEFAULT_LIMIT))
candidates: List[Dict[str, Any]] = []
sources: Dict[str, str] = {}
crypto_pair = _canonical_crypto_pair(query)
connector_hits, connector_source, connector_status = (
_search_selected_connector(query, limit)
)
if connector_source is not None and connector_status is not None:
sources[connector_source] = connector_status
candidates.extend(connector_hits)
if connector_source == "mt5":
# Broker-native identities must never be substituted by web search.
return json.dumps({
"ok": True, "market": "multi", "source": "symbol_search",
"data": {"query": query, "count": len(connector_hits[:limit]),
"candidates": connector_hits[:limit], "sources": sources},
}, ensure_ascii=False)
if crypto_pair is not None and not connector_hits:
# Resolving a pair must not require a broker account. The venue
# catalogs are public, unauthenticated REST — the same connectivity
# `orderbook_depth` already uses to serve these very pairs — so a
# user with no Binance connection still gets an identity instead of
# nothing (or, before #1234, a near-string Yahoo asset).
public_hits, public_source, public_status = _search_public_exchanges(
crypto_pair
)
if public_source is not None:
sources[public_source] = public_status or "ok"
candidates.extend(public_hits)
em_hits, sources["eastmoney"] = _search_eastmoney(query)
candidates.extend(em_hits)
# An explicit FX pair searches Yahoo by its canonical ``XXXYYY=X``
# spelling — exact-symbol search is far more reliable than free text —
# and always yields a deterministic candidate, so a throttled/outage
# Yahoo (the earlier "GBP/USD -> 0 candidates" failure) never turns a
# canonical pair into nothing.
fx_pair = _canonical_fx_pair(query)
yh_hits, sources["yahoo"] = _search_yahoo(fx_pair or query)
candidates.extend(yh_hits)
if fx_pair is not None:
pair_no_x = fx_pair[:-2]
candidates.append(
{
"symbol": fx_pair,
"name": f"{pair_no_x[:3]}/{pair_no_x[3:]}",
"market": "fx",
"type": "currency",
"exchange": "CCY",
"source": "fx_normalizer",
}
)
sources["fx_normalizer"] = "ok"
if crypto_pair is not None:
# A pair query is an exact instrument assertion. Near-string Yahoo
# hits such as AETHUSDT-USD are different assets and must not enter
# the identity ledger as rival candidates (#1234).
candidates = [
candidate
for candidate in candidates
if _canonical_crypto_pair(str(candidate.get("symbol") or ""))
== crypto_pair
]
elif _metal_or_fx_legs(query) is not None:
# A spot metal / FX pair query (``XAUUSD``, ``XAU/USD``,
# ``XAUUSD=X``) is an exact instrument assertion, exactly like the
# crypto-pair branch above. Yahoo answers such a query by
# free-text similarity, and its top hit for spot gold is a Swedish
# Bitcoin ETP (``VALOUR-BTC-0-SEK.ST``) or an Aave token
# (``AETHUSDT-USD``) — a wrong instrument that then locks the run's
# identity. Keep only candidates naming the same two legs; an
# empty candidate list leaves the identity unresolved, which is
# the safe failure.
query_legs = _metal_or_fx_legs(query)
candidates = [
candidate
for candidate in candidates
if _metal_or_fx_legs(str(candidate.get("symbol") or "")) == query_legs
]
elif query.strip().upper().endswith("=F"):
# Yahoo's continuous-front-month futures notation is likewise an
# exact assertion: ``GC=F`` is gold futures, and a near-string hit
# is a different contract.
candidates = [
candidate
for candidate in candidates
if str(candidate.get("symbol") or "").strip().upper()
== query.strip().upper()
]
if (crypto_pair is None and _metal_or_fx_legs(query) is None
and not query.strip().upper().endswith("=F")
and not query.startswith("^") and not _is_canadian_symbol(query)):
from src.tools.gildata_identity import search_candidates
gildata_hits, gildata_status = search_candidates(query, limit)
if gildata_status is not None:
sources["gildata"] = gildata_status
candidates.extend(gildata_hits)
# Canada fail-fast: a Canadian ticker must resolve to the Canadian venue
# only. Yahoo also returns the US OTC alias of the same company (e.g.
# ``BYN.V`` -> ``BYAGF.US``), which would make the grounding ledger see
# two venues for one entity and reject every downstream call with
# ``identity_conflict``. Keep only ``.TO``/``.V`` candidates here.
if _is_canadian_symbol(query):
candidates = [
c
for c in candidates
if _is_canadian_symbol(str(c.get("symbol") or ""))
]
merged = _merge_candidates(candidates)
merged, sources["sec_edgar"] = _enrich_us_cik(merged)
if sources["sec_edgar"] == _NO_US:
del sources["sec_edgar"]
merged = merged[:limit]
return json.dumps(
{
"ok": True,
"market": "multi",
"source": "symbol_search",
"data": {
"query": query,
"count": len(merged),
"candidates": merged,
"sources": sources,
},
},
ensure_ascii=False,
)
def _clamp_limit(value: Any) -> int:
"""Coerce a requested count into the supported ``1.._MAX_LIMIT`` range."""
try:
n = int(value)
except (TypeError, ValueError, OverflowError):
return _DEFAULT_LIMIT
return max(1, min(n, _MAX_LIMIT))
def _is_canadian_symbol(text: str) -> bool:
"""Whether *text* is a Canadian ticker (TSX ``.TO`` / TSXV ``.V``).
Used for fail-fast routing: Canadian symbols are served by Yahoo only, so
Eastmoney is skipped and US OTC aliases are filtered from the result set.
Args:
text: A symbol or free-text query to test.
Returns:
``True`` when the text starts with a Canadian-suffixed ticker
(``BTO.TO``, ``BTO.TO B2Gold``, ``SGML.V Sigma Lithium``, ...).
"""
return bool(_CANADIAN_SYMBOL_RE.match((text or "").strip()))
def _canonical_crypto_pair(value: str) -> str | None:
"""Return an explicit crypto pair in canonical ``BASE-QUOTE`` form.
A pair is only accepted as a crypto pair when its base is in the
appropriate whitelist. Stablecoin quotes (``USDT``/``USDC``/``BUSD``/
``TUSD``/``FDUSD``) accept any alphanumeric base — a 6-letter base
cannot be confused with a stablecoin because no real-world asset
except crypto ones trades quoted in stablecoins. The ``USD`` quote
is gated on :data:`_CRYPTO_USD_BASES` so a bare ``XAUUSD`` /
``EURUSD`` / ``GBPUSD`` query does NOT auto-lock onto tokenized gold
or a forex pair that the public venue catalogs do not list.
Returns ``"BASE-QUOTE"`` for an accepted pair, ``None`` otherwise.
"""
clean = str(value or "").strip().upper()
matched = _CRYPTO_PAIR_RE.fullmatch(clean)
if matched:
base, quote = matched.group(1), matched.group(2)
# Two independent rejections, both required. The fiat/fiat rule (from
# main) covers pairs whose quote leg is not USD; the USD whitelist
# (this PR) covers a USD quote whose base is not a known crypto —
# XAU is not a fiat code, so fiat/fiat alone lets XAU-USD through and
# the venue catalog locks tokenized gold as "spot gold".
if base in FIAT_CODES and quote in FIAT_CODES:
return None # fiat/fiat is an FX pair, not crypto
if quote == "USD" and base not in _CRYPTO_USD_BASES:
return None
return f"{base}-{quote}"
if clean.isalnum():
for quote in _CRYPTO_QUOTE_ASSETS:
if clean.endswith(quote) and len(clean) > len(quote) + 1:
base = clean[: -len(quote)]
# Same two rejections. They differ deliberately in kind:
# fiat/fiat has no crypto reading at all, so it gives up;
# a non-whitelisted USD base only rules out THIS quote asset,
# so it must `continue` and let a longer quote (USDT/USDC)
# still match — returning here would strand e.g. XAUT-USDT.
if base in FIAT_CODES and quote in FIAT_CODES:
return None # fiat/fiat is an FX pair, not crypto
if quote == "USD" and base not in _CRYPTO_USD_BASES:
continue
return f"{base}-{quote}"
return None
#: Public, no-auth venue catalogs consulted for an explicit pair, in order.
#: Same venues and same ccxt connectivity as ``orderbook_depth``.
_PUBLIC_CRYPTO_EXCHANGES = ("binance", "okx")
def _load_public_markets(exchange_id: str) -> Dict[str, Any]:
"""Return one venue's public market catalog via ccxt (no credentials).
Isolated as its own function so tests monkeypatch exactly this name and
never open a socket, the same pattern as
``orderbook_depth_tool._fetch_raw_book``.
Args:
exchange_id: A ccxt exchange id, ``"binance"`` or ``"okx"``.
Returns:
ccxt's unified markets mapping, keyed by ``BASE/QUOTE``.
Raises:
Exception: Whatever ccxt raises for a network or venue error.
"""
import ccxt
exchange = getattr(ccxt, exchange_id)({"enableRateLimit": True, "timeout": 10_000})
return exchange.load_markets()
def _search_public_exchanges(
crypto_pair: str,
) -> tuple[List[Dict[str, Any]], str | None, str | None]:
"""Resolve an exact pair against the public venue catalogs.
Args:
crypto_pair: Canonical ``BASE-QUOTE`` spelling.
Returns:
``(candidates, source, status)``; ``source`` is ``None`` only when
ccxt itself is unavailable.
"""
base, quote = crypto_pair.split("-", 1)
ccxt_symbol = f"{base}/{quote}"
failures: List[str] = []
for exchange_id in _PUBLIC_CRYPTO_EXCHANGES:
try:
markets = _load_public_markets(exchange_id)
except ImportError:
return [], None, None
except Exception as exc: # noqa: BLE001 — one venue is non-fatal
logger.debug("public %s catalog failed for %r: %s", exchange_id, crypto_pair, exc)
failures.append(f"{exchange_id}: {exc}")
continue
market = markets.get(ccxt_symbol) if isinstance(markets, dict) else None
if not isinstance(market, dict) or market.get("active") is False:
continue
if market.get("spot") is False:
continue
return (
[
{
"symbol": crypto_pair,
"name": None,
"market": "crypto",
"type": "cryptocurrency",
"exchange": exchange_id.upper(),
"source": "public_exchange",
}
],
"public_exchange",
"ok",
)
if failures:
return [], "public_exchange", "; ".join(failures)
return [], "public_exchange", f"{_SKIPPED}no public venue lists {crypto_pair}"
def _search_selected_connector(
query: str,
limit: int,
) -> tuple[List[Dict[str, Any]], str | None, str | None]:
"""Resolve an explicit pair against the active crypto connector, if supported."""
crypto_pair = _canonical_crypto_pair(query)
mt5_pair = _metal_or_fx_legs(query)
# Lazy imports keep the generic symbol tool usable when optional connector
# dependencies are absent and avoid loading broker configuration at import.
from src.trading import profiles as trading_profiles
from src.trading import service as trading_service
try:
profile_id = trading_profiles.load_selected_profile_id()
profile = trading_profiles.profile_by_id(profile_id)
except (OSError, ValueError) as exc:
logger.debug("selected connector lookup failed for %r: %s", query, exc)
return [], None, None
if profile.connector == "binance" and crypto_pair is None:
return [], None, None
if profile.connector not in {"binance", "mt5"}:
return [], None, None
source = profile.connector
try:
payload = trading_service.search_instruments(
query,
profile.id,
limit=limit,
)
except Exception as exc: # noqa: BLE001 - one search source is non-fatal
logger.debug("%s instrument search failed (%s)", source, type(exc).__name__)
message = "connector search failed" if source == "mt5" else f"connector search failed: {exc}"
return [], source, message
if not isinstance(payload, dict) or str(payload.get("status")).casefold() != "ok":
message = (
str(payload.get("error") or payload.get("message") or "unknown error")
if isinstance(payload, dict)
else "invalid response"
)
return [], source, f"connector search failed: {message}"
rows = payload.get("instruments")
rows = rows if isinstance(rows, list) else []
candidates: List[Dict[str, Any]] = []
if profile.connector == "mt5":
requested_base = "".join(mt5_pair) if mt5_pair else query.strip().upper()
for row in rows:
if not isinstance(row, dict):
continue
native_symbol = str(
row.get("native_symbol") or row.get("symbol") or ""
).strip()
resolved_base = native_symbol.upper()
if resolved_base.endswith(".FX"):
resolved_base = resolved_base[:-3]
if not (resolved_base.startswith(requested_base) if mt5_pair else resolved_base == requested_base):
continue
candidates.append(
{
"symbol": query.strip().upper() if mt5_pair else native_symbol,
"name": str(row.get("name") or native_symbol).strip() or None,
"market": str(row.get("market") or "mt5"),
"type": str(row.get("type") or "cfd"),
"exchange": str(row.get("exchange") or row.get("venue") or "MT5"),
"venue": str(row.get("venue") or row.get("exchange") or "MT5"),
"source": source,
"profile_id": profile.id,
"native_symbol": native_symbol,
}
)
if len({candidate["native_symbol"] for candidate in candidates}) > 1:
return [], source, "ambiguous broker symbol; specify an exact native symbol"
return candidates, source, "ok"
for row in rows:
if not isinstance(row, dict):
continue
symbol = _canonical_crypto_pair(str(row.get("symbol") or ""))
if symbol is None:
continue
candidates.append(
{
"symbol": symbol,
"name": str(row.get("name") or row.get("native_symbol") or "").strip()
or None,
"market": "crypto",
"type": str(row.get("type") or "cryptocurrency"),
"exchange": str(row.get("exchange") or source).upper(),
"source": source,
"profile_id": profile.id,
}
)
return candidates, source, "ok"
def _is_ticker_name_query(query: str) -> bool:
"""Whether *query* is a bare all-caps ticker followed by a name hint.
Yahoo's search endpoint answers this shape ("XOM ExxonMobil") with zero
quotes, so the Yahoo path skips it rather than letting a caller deciding
whether an entity exists read the empty result as "not listed". Unlike the
Canadian fail-fast, Eastmoney is NOT skipped for this shape — it can serve
multi-token queries — so this helper is only consulted on the Yahoo path.
The first token must be bare all-caps (``[A-Z0-9&]{1,6}``); the absence of
a dot/hyphen excludes suffixed tickers (``BTO.TO``, ``BRK.B``) and
mixed-case name tokens.
Args:
query: Free-text name or ticker fragment.
Returns:
``True`` when the query has at least two tokens and starts with a
bare all-caps ticker-like token.
"""
tokens = (query or "").strip().split()
return len(tokens) >= 2 and bool(re.fullmatch(r"[A-Z0-9&]{1,6}", tokens[0]))
def _search_eastmoney(query: str) -> tuple[List[Dict[str, Any]], str]:
"""Query Eastmoney's suggest endpoint and normalize the candidates.
Fails fast for Canadian (``.TO``/``.V``) queries: Eastmoney has no Canada
coverage and its suggest endpoint returns a non-JSON body for those,
so the endpoint is skipped instead of raising a parse error.
Args:
query: Free-text name or ticker fragment.
Returns:
``(candidates, status)`` where ``status`` is ``"ok"`` on success or a
short error string when the source failed (candidates is then empty).
"""
if _canonical_crypto_pair(query) is not None:
return [], f"{_SKIPPED}eastmoney has no crypto exchange-pair coverage"
if _is_canadian_symbol(query):
logger.info(
"eastmoney skipped for Canadian symbol %r (no Canada coverage)",
query,
)
return [], f"{_SKIPPED}eastmoney has no Canada coverage"
try:
payload = eastmoney_client.get_json(
_EASTMONEY_SUGGEST_URL,
params={"input": query, "type": "14", "count": str(_PER_SOURCE_CAP)},
)
except Exception as exc: # noqa: BLE001 - one source failing is non-fatal
# Deliberately debug-level, not warning: Eastmoney has no coverage for
# many queries the fan-out legitimately tries (Canadian names, crypto,
# futures) and returns a non-JSON body for them. That is expected and
# benign — the status string below still flows to the tool result so
# nothing is hidden, it just no longer spams the terminal. Failures on
# queries Eastmoney SHOULD cover (A-share/HK) are still visible by
# checking the tool result's sources map or with debug logging on.
logger.debug("eastmoney suggest failed for %r: %s", query, exc)
return [], f"eastmoney search failed: {exc}"
rows = _eastmoney_data_rows(payload)
candidates = [c for c in (_eastmoney_candidate(r) for r in rows) if c is not None]
return candidates, "ok"
def _eastmoney_data_rows(payload: Any) -> List[Dict[str, Any]]:
"""Extract the ``QuotationCodeTable.Data`` rows from a suggest payload."""
if not isinstance(payload, dict):
return []
table = payload.get("QuotationCodeTable")
if not isinstance(table, dict):
return []
data = table.get("Data")
if not isinstance(data, list):
return []
return [row for row in data if isinstance(row, dict)]
def _eastmoney_candidate(row: Dict[str, Any]) -> Optional[Dict[str, Any]]:
"""Map one Eastmoney suggest row to a normalized candidate, or ``None``.
Eastmoney rows carry ``QuoteID`` (``<market>.<code>``), ``Code``, ``Name``,
``MktNum`` and ``SecurityTypeName``. A row whose market we cannot map to a
project suffix is dropped rather than emitted with a wrong symbol.
Args:
row: One ``QuotationCodeTable.Data`` element.
Returns:
A candidate dict, or ``None`` when the row is unusable.
"""
quote_id = row.get("QuoteID")
market = ""
code = str(row.get("Code") or "").strip()
if isinstance(quote_id, str) and "." in quote_id:
market, _, qid_code = quote_id.partition(".")
code = code or qid_code.strip()
else:
market = str(row.get("MktNum") or "").strip()
suffix = _EASTMONEY_SUFFIX_BY_MARKET.get(market)
if not suffix or not code:
return None
# Market 0 also carries the Beijing exchange: 920xxx (current) and
# legacy 4xxxxx/8xxxxx codes. Shenzhen codes start with 0/2/3.
if suffix == "SZ" and len(code) == 6 and code.startswith(("4", "8", "92")):
suffix = "BJ"
symbol = _format_symbol(code, suffix)
if symbol is None:
return None
name = str(row.get("Name") or "").strip() or None
sec_type = str(row.get("SecurityTypeName") or "").strip() or None
return {
"symbol": symbol,
"name": name,
"market": _MARKET_BY_SUFFIX.get(suffix, suffix.lower()),
"type": sec_type,
"source": "eastmoney",
}
def _format_symbol(code: str, suffix: str) -> Optional[str]:
"""Render a bare code + suffix into the project symbol convention.
HK codes are zero-padded to five digits to match the loader/secid scheme.
Args:
code: Bare instrument code (e.g. ``"600519"``, ``"700"``, ``"AAPL"``).
suffix: One of ``SH``/``SZ``/``BJ``/``HK``/``US``.
Returns:
The formatted symbol (``"600519.SH"``, ``"00700.HK"``, ``"AAPL.US"``),
or ``None`` when the code is empty.
"""
code = code.strip().upper()
if not code:
return None
if suffix == "HK":
return f"{code.zfill(5)}.HK"
return f"{code}.{suffix}"
def _search_yahoo(query: str) -> tuple[List[Dict[str, Any]], str]:
"""Query Yahoo's search endpoint and normalize the quote candidates.
Args:
query: Free-text name or ticker fragment.
Returns:
``(candidates, status)`` where ``status`` is ``"ok"`` on success, a
``"skipped: ..."`` marker when the source cannot serve this query shape,
or a short error string when the source failed (candidates is then
empty).
"""
if not query.isascii():
# Yahoo's search endpoint answers any non-ASCII query with HTTP 400
# (verified against both query1 and query2 hosts), so calling it spends
# a request to manufacture a source failure. That failure is not
# cosmetic: a caller deciding whether an entity exists counts clean
# sources, and an unsupported query shape must not read as an outage.
return [], f"{_SKIPPED}non-ASCII query is not supported by this source"
try:
quotes = yahoo_client.search(query)
except Exception as exc: # noqa: BLE001 - one source failing is non-fatal
logger.warning("yahoo search failed for %r: %s", query, exc)
return [], f"yahoo search failed: {exc}"
if not quotes and _is_ticker_name_query(query):
# Mirror the non-ASCII guard above: Yahoo answers a multi-token
# ticker+name query ("XOM ExxonMobil") with zero quotes, which a
# caller deciding whether an entity exists would otherwise read as
# "not listed". An unsupported query shape must not read as an outage.
return [], f"{_SKIPPED}ticker+name query is not supported by this source"
candidates: List[Dict[str, Any]] = []
for quote in quotes[:_PER_SOURCE_CAP]:
candidate = _yahoo_candidate(quote)
if candidate is not None:
candidates.append(candidate)
return candidates, "ok"
def _yahoo_candidate(quote: Dict[str, Any]) -> Optional[Dict[str, Any]]:
"""Map one Yahoo search quote to a normalized candidate, or ``None``.
Yahoo carries US tickers bare, HK tickers as ``0700.HK``, and Canadian
listings with ``.TO`` / ``.V`` suffixes. We translate those into the project
convention (``AAPL.US`` / ``00700.HK`` / ``TD.TO`` / ``PNG.V``) and leave
other instruments (crypto, indices, FX) on their native Yahoo symbol.
Args:
quote: One element of Yahoo search's ``quotes`` list.
Returns:
A candidate dict, or ``None`` when the quote has no symbol.
"""
raw_symbol = str(quote.get("symbol") or "").strip()
if not raw_symbol:
return None
symbol, market = _from_yahoo_symbol(raw_symbol, quote)
name = (
str(quote.get("shortname") or quote.get("longname") or "").strip() or None
)
return {
"symbol": symbol,
"name": name,
"market": market,
"type": str(quote.get("quoteType") or "").strip().lower() or None,
"exchange": str(quote.get("exchange") or "").strip() or None,
"source": "yahoo",
}
def _from_yahoo_symbol(raw_symbol: str, quote: Dict[str, Any]) -> tuple[str, str]:
"""Translate a Yahoo symbol into the project convention + market label.
Args:
raw_symbol: The Yahoo-side symbol (e.g. ``AAPL``, ``0700.HK``,
``TD.TO``, ``PNG.V``, or ``BTC-USD``).
quote: The full Yahoo quote, used to classify the instrument via
``quoteType`` (bare US equity/ETF/fund vs crypto/index/FX) and to
keep a bare symbol filed under a non-US venue on its native form
via ``exchange``.
Returns:
``(symbol, market)`` in the project convention.
"""
upper = raw_symbol.upper()
if upper.endswith(".HK"):
base = raw_symbol[: -len(".HK")].lstrip("0") or "0"
return f"{base.zfill(5)}.HK", "hk"
# Yahoo quotes Shanghai as ``.SS`` where this project (and Eastmoney) use
# ``.SH``. Emitting both spellings published one listing as two rival
# candidates, which the identity gate could not choose between, so every
# Shanghai query dead-ended as ambiguous. Folding here also lets the two
# sources merge and corroborate each other via ``also_from``.
if upper.endswith(".SS"):
return f"{upper[: -len('.SS')]}.SH", "cn"
suffix = upper.rsplit(".", 1)[-1] if "." in upper else ""
if suffix in _MARKET_BY_SUFFIX:
return upper, _MARKET_BY_SUFFIX[suffix]
quote_type = str(quote.get("quoteType") or "").strip().upper()
if quote_type == "CURRENCY":
# FX pairs canonicalize to the ``XXXYYY=X`` form the fetch layer
# serves directly (``GBP/USD`` -> ``GBPUSD=X``); non-fiat currency
# quotes (metals like XAU/USD) keep their native symbol.
canon = _canonical_fx_pair(raw_symbol)
if canon is not None:
return canon, "fx"
return raw_symbol, "global"
if raw_symbol.startswith("^"):
return raw_symbol, "index"
exchange = str(quote.get("exchange") or "").strip().upper()
if (
quote_type in _US_LISTED_QUOTE_TYPES
and "." not in raw_symbol
and "-" not in raw_symbol
and (not exchange or exchange in _US_YAHOO_EXCHANGES)
):
# ETFs and mutual funds are US listings just like equities: Yahoo
# carries them bare (``QQQ``, ``SPY``), and the fetch layer already
# routes a bare ticker as ``us_equity``. Leaving them on Yahoo's
# native symbol with market ``global`` split the two layers — the
# resolver locked ``QQQ`` while ``get_market_data`` demands
# ``QQQ.US``, so every ETF question ended degraded (#2084).
return f"{upper}.US", "us"
# Crypto, indices, FX, ETFs on non-US exchanges: keep Yahoo's native symbol.
return raw_symbol, "global"
def _merge_candidates(candidates: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
"""De-duplicate candidates by symbol, preserving first-seen order.
When two sources resolve the same symbol the first hit wins and the second
source name is appended to a ``also_from`` list so provenance is not lost.
Args:
candidates: Raw candidates from every source, in fan-out order.
Returns:
A de-duplicated candidate list (immutable inputs are copied, not mutated).
"""
by_symbol: Dict[str, Dict[str, Any]] = {}
order: List[str] = []
for candidate in candidates:
symbol = candidate.get("symbol")
if not symbol:
continue
if symbol not in by_symbol:
by_symbol[symbol] = dict(candidate)
order.append(symbol)
continue
existing = by_symbol[symbol]
other = candidate.get("source")
if other and other != existing.get("source"):
also = list(existing.get("also_from") or [])
if other not in also:
also.append(other)
merged = dict(existing)
merged["also_from"] = also
# Backfill a missing name from the duplicate hit.
if not merged.get("name") and candidate.get("name"):
merged["name"] = candidate["name"]
by_symbol[symbol] = merged
return [by_symbol[sym] for sym in order]
def _enrich_us_cik(
candidates: List[Dict[str, Any]],
) -> tuple[List[Dict[str, Any]], str]:
"""Return new candidates with a SEC CIK attached to U.S.-equity rows.
Only ``.US`` equity symbols are looked up; the SEC table maps bare tickers
to a zero-padded 10-digit CIK. A lookup failure stops further lookups and is
reported via the status; resolved CIKs found before it still apply.
Args:
candidates: Merged candidate rows (left unmodified).
Returns:
``(new_candidates, status)`` where ``status`` is :data:`_NO_US` when no
U.S. equity was present, ``"ok"`` on a clean pass, or a short error
string when a SEC lookup failed.
"""
has_us = any(
isinstance(c.get("symbol"), str) and c["symbol"].upper().endswith(".US")
for c in candidates
)
if not has_us:
return candidates, _NO_US
status = "ok"
out: List[Dict[str, Any]] = []
for candidate in candidates:
symbol = candidate.get("symbol")
if status == "ok" and isinstance(symbol, str) and symbol.upper().endswith(".US"):
ticker = symbol[: -len(".US")]
try:
cik = sec_edgar_client.cik_for(ticker)
except Exception as exc: # noqa: BLE001 - enrichment failure is non-fatal
logger.warning("sec cik_for failed for %s: %s", ticker, exc)
status = f"sec lookup failed: {exc}"
out.append(candidate)
continue
if cik:
out.append({**candidate, "cik": cik})
continue
out.append(candidate)
return out, status
def _error(message: str) -> str:
"""Render a failure envelope as a JSON string."""
return json.dumps({"ok": False, "error": message}, ensure_ascii=False)