"Vibe-Trading: Your Personal Trading Agent"
Python ai-agentalgorithmic-tradingbacktestingfintechllmmcpmulti-agentpythonquantitative-financetrading
fix(grounding): keep tail-risk measure and confidence through the gate
Yufeng He452c64db1139
diff --git a/agent/src/agent/grounding/evidence.py b/agent/src/agent/grounding/evidence.pyindex 5c6fe8157..f003a3256 100644--- a/agent/src/agent/grounding/evidence.py+++ b/agent/src/agent/grounding/evidence.py@@ -123,7 +123,11 @@ _ANALYSIS_KIND_ALIASES = {"var_95": "tail_risk","var_99": "tail_risk","cvar": "tail_risk",+ "cvar_95": "tail_risk",+ "cvar_99": "tail_risk","es": "tail_risk",+ "es_95": "tail_risk",+ "es_99": "tail_risk","expected_shortfall": "tail_risk",# Chinese TOOL FIELD NAMES from A-share tools, not answer prose."最大回撤": "drawdown",@@ -357,17 +361,16 @@ _QUALIFIER_SUFFIXES = frozenset()-def _metric_kind_for_path(path: str) -> str | None:- """Map an evidence JSON path to an analysis metric kind.+def _metric_alias_key_for_path(path: str) -> str | None:+ """The alias-table key an evidence JSON path resolves to, or None.- A metadata count leaf (:func:`_is_metadata_count_leaf`) has no kind.+ A metadata count leaf (:func:`_is_metadata_count_leaf`) resolves to nothing."""if _is_metadata_count_leaf(path):return Noneleaf = _leaf_name(path)- kind = _ANALYSIS_KIND_ALIASES.get(leaf)- if kind is not None:- return kind+ if leaf in _ANALYSIS_KIND_ALIASES:+ return leaf# Compound leaves ("strategy_max_drawdown"): scan tokens from the right,# where English puts the head noun, so "return_vol" is vol, not return.# Only the head of a compound leaf says what it measures (#1426):@@ -383,12 +386,60 @@ def _metric_kind_for_path(path: str) -> str | None:if len(candidate) < size:continuekey = "_".join(candidate[-size:])- kind = None if key in _EXACT_ONLY_ALIASES else _ANALYSIS_KIND_ALIASES.get(key)- if kind is not None:- return kind+ if key in _EXACT_ONLY_ALIASES:+ continue+ if key in _ANALYSIS_KIND_ALIASES:+ return keyreturn None+def _metric_kind_for_path(path: str) -> str | None:+ """Map an evidence JSON path to an analysis metric kind.++ A metadata count leaf (:func:`_is_metadata_count_leaf`) has no kind.+ """+ key = _metric_alias_key_for_path(path)+ return _ANALYSIS_KIND_ALIASES.get(key) if key is not None else None+++# What a tail-risk measure word reports: an ES and a VaR are different+# measurements even when they share a confidence level and a number.+_TAIL_RISK_MEASURES = {+ "var": "var",+ "cvar": "es",+ "es": "es",+ "expected_shortfall": "es",+}+++def _tail_risk_identity_for_path(path: str) -> tuple[str, int | None] | None:+ """The ``(measure, confidence)`` a tail-risk evidence path reports.++ ``var_95`` is a 95% VaR: not an ES, and not a 99% anything. A numeric token+ right after the measure head is the confidence; names without one carry+ ``None`` and can only ground a claim that states no confidence.+ """+ key = _metric_alias_key_for_path(path)+ if key is None or _ANALYSIS_KIND_ALIASES.get(key) != "tail_risk":+ return None+ if key in _TAIL_RISK_MEASURES:+ # A compound leaf can still carry the confidence after the head:+ # "portfolio_cvar_95" resolves as "cvar", with 95 stripped as a+ # qualifier. Recover it from the leaf itself.+ tokens = [token for token in re.split(r"[_.]", _leaf_name(path)) if token]+ for i in range(len(tokens) - 1, -1, -1):+ head_len = 2 if "_".join(tokens[i : i + 2]) == key else 1 if tokens[i] == key else 0+ if head_len:+ following = tokens[i + head_len] if i + head_len < len(tokens) else ""+ return _TAIL_RISK_MEASURES[key], int(following) if following.isdigit() else None+ return _TAIL_RISK_MEASURES[key], None+ head, _, tail = key.rpartition("_")+ measure = _TAIL_RISK_MEASURES.get(head)+ if measure is None or not tail.isdigit():+ return None+ return measure, int(tail)++@dataclass(frozen=True)class EvidenceRecord:"""One observed, unavailable, or derived numeric evidence item."""diff --git a/agent/src/agent/grounding/figures.py b/agent/src/agent/grounding/figures.pyindex 3557e8fcf..067aef36d 100644--- a/agent/src/agent/grounding/figures.py+++ b/agent/src/agent/grounding/figures.py@@ -17,7 +17,7 @@ import htmlimport reimport stringimport unicodedata-from dataclasses import dataclass+from dataclasses import dataclass, replacefrom decimal import Decimal, InvalidOperationfrom typing import Iterable, Sequence@@ -180,6 +180,11 @@ class Figure:extent: tuple[int, int] | None = None# The info string of the fenced block holding the figure, or None.fence: str | None = None+ # The tail-risk frame the figure sits in ("VaR 95%: 1.57%"): the measured+ # value carries (measure, confidence); the confidence figure itself is+ # flagged instead and never checked as a measurement.+ tail_risk: tuple[str, int | None] | None = None+ tail_risk_confidence: bool = False@dataclass(frozen=True)@@ -859,6 +864,97 @@ def _short_date_is_structural(content: str, match: re.Match[str], full_dates: Sereturn any(segment <= full.start() and full.end() <= match.start() for full in full_dates)+# A tail-risk claim's prose frame: the measure word, then an optional+# confidence level. VaR and ES/CVaR are different measurements at any+# confidence, and a 95% and a 99% VaR are different ones.+_TAIL_RISK_WORD_RE = re.compile(+ r"(?<![A-Za-z])(?:C?VaR|ES)(?![A-Za-z])|(?i:expected shortfall)|在险价值|风险价值|预期损失|条件在险价值"+)++# The confidence sits right after the measure word ("VaR 95%", "VaR (95%)",+# "ES at 99%") or right before it ("95% VaR", "95% 置信水平下的 VaR").+_TAIL_RISK_CONFIDENCE_AFTER_RE = re.compile(+ r"\s*[((]?\s*(?:at|of)?\s*(\d{1,3}(?:\.\d+)?)\s*[%%]"+)+_TAIL_RISK_CONFIDENCE_BEFORE_RE = re.compile(+ r"(\d{1,3}(?:\.\d+)?)\s*[%%]\s*(?:置信水平|置信度|confidence(?:\s+level)?)?\s*(?:下|中|的)?\s*[,,]?\s*$"+)+++def _tail_risk_measure_of(word: str) -> str:+ """The measure a tail-risk keyword reports: ES and CVaR are one family."""+ return (+ "es"+ if word.casefold() == "expected shortfall" or word in ("ES", "CVaR", "预期损失", "条件在险价值")+ else "var"+ )+++def _tail_risk_frames(content: str, figures: list[Figure]) -> list[Figure]:+ """Mark the tail-risk frame each percent figure sits in, when there is one.++ The confidence level in "VaR 95%: 1.57%" is part of the frame, not a+ measurement: it is flagged so the gate never checks it, while the value is+ checked only against evidence of the same measure and confidence.+ """+ if not any(figure.percent for figure in figures):+ return figures+ lines = _lines_with_offsets(content)+ out = list(figures)+ for line_index, (line_text, line_start) in enumerate(lines):+ line_figures = [+ (index, figure)+ for index, figure in enumerate(figures)+ if figure.line == line_index and figure.percent+ ]+ if not line_figures:+ continue+ keywords = list(_TAIL_RISK_WORD_RE.finditer(line_text))+ if not keywords:+ continue+ # A frame runs from its keyword to the next keyword or the clause end:+ # "VaR 95%: 1,57%. Drawdown máximo −5,132%" keeps the drawdown out of+ # the VaR frame entirely.+ terminators = [+ match.end()+ for match in re.finditer(r"[。!!??;;.]", line_text)+ ]+ regions: list[tuple[int, int]] = []+ for pos, keyword in enumerate(keywords):+ end = keywords[pos + 1].start() if pos + 1 < len(keywords) else len(line_text)+ boundary = next((mark for mark in terminators if mark > keyword.end()), end)+ regions.append((keyword.start(), min(end, boundary)))+ # (measure, confidence, confidence digit start in source coords, region)+ frames: list[tuple[str, int | None, int | None, tuple[int, int]]] = []+ for keyword, region in zip(keywords, regions):+ measure = _tail_risk_measure_of(keyword.group(0))+ confidence: int | None = None+ confidence_start: int | None = None+ match = _TAIL_RISK_CONFIDENCE_AFTER_RE.match(line_text, keyword.end())+ if match is None:+ match = _TAIL_RISK_CONFIDENCE_BEFORE_RE.search(line_text, 0, keyword.start())+ if match:+ confidence = int(float(match.group(1)))+ confidence_start = line_start + match.start(1)+ frames.append((measure, confidence, confidence_start, region))+ for index, figure in line_figures:+ if figure.start in {frame[2] for frame in frames}:+ out[index] = replace(figure, tail_risk_confidence=True)+ continue+ line_pos = figure.start - line_start+ region = next(+ (frame[3] for frame in frames if frame[3][0] <= line_pos < frame[3][1]),+ None,+ )+ if region is None:+ continue+ measure, confidence, _, _ = next(+ frame for frame in frames if frame[3] == region+ )+ out[index] = replace(figure, tail_risk=(measure, confidence))+ return out++def scan_figures(content: str, block: FiguresBlock) -> list[Figure]:"""Locate and classify every number in the prose of a draft (spec §3).@@ -959,4 +1055,4 @@ def scan_figures(content: str, block: FiguresBlock) -> list[Figure]:),))- return figures+ return _tail_risk_frames(content, figures)diff --git a/agent/src/agent/grounding/policies.py b/agent/src/agent/grounding/policies.pyindex 411a33f12..5daf14402 100644--- a/agent/src/agent/grounding/policies.py+++ b/agent/src/agent/grounding/policies.py@@ -25,6 +25,7 @@ from src.agent.grounding.evidence import (_is_price_kind,_metric_kind_for_path,_price_field_for_path,+ _tail_risk_identity_for_path,_timestamp_matches_claim_date,)from src.agent.grounding.figures import (@@ -466,6 +467,11 @@ class _PolicyMixin:if figure.shape not in ("measured", "bare"):continuedeclaration = block.match(figure.value, figure.percent)+ if figure.tail_risk_confidence and declaration is None:+ # The confidence level of a tail-risk frame ("VaR 95%: 1.57%")+ # qualifies the measurement; it is not one itself. A declared+ # role wins: the model said what the number is.+ continuesymbol = self._figure_symbol(content, figure, declaration, line_symbols, document_symbol, records)@@ -849,12 +855,41 @@ class _PolicyMixin:return low * 0.9 <= value <= high * 1.1return low >= _INTEGER_PRICE_FLOOR and low * 0.5 <= value <= high * 2.0- def _metric_pool(self, symbol: str | None) -> list[float]:- """Metric values from completed analysis results and metric-named leaves."""+ @staticmethod+ def _tail_risk_compatible(+ field_name: str, measure: str, confidence: int | None+ ) -> bool:+ """Whether a tail-risk evidence field can ground a claim's identity.++ The measure must agree; a confidence the claim states must be matched+ exactly, while a claim stating none takes any evidence of its measure.+ """+ identity = _tail_risk_identity_for_path(field_name)+ if identity is None or identity[0] != measure:+ return False+ return confidence is None or identity[1] == confidence++ def _metric_pool(+ self,+ symbol: str | None,+ *,+ tail_risk: tuple[str, int | None] | None = None,+ ) -> list[float]:+ """Metric values from completed analysis results and metric-named leaves.++ A tail-risk claim narrows the pool to evidence of its own measure and,+ when the claim states one, its confidence: a ``var_95`` reading says+ nothing about a "VaR 99%" or an "ES 95%" figure.+ """+ measure, confidence = tail_risk if tail_risk is not None else (None, None)values = [float(entry["value"])for entry in self._analysis_metricsif entry.get("value") is not None+ and (+ measure is None+ or self._tail_risk_compatible(str(entry.get("field") or ""), measure, confidence)+ )]values.extend(float(record.value)@@ -863,6 +898,10 @@ class _PolicyMixin:and record.value is not Noneand _metric_kind_for_path(record.field) is not Noneand (not symbol or not record.symbol or record.symbol == symbol)+ and (+ measure is None+ or self._tail_risk_compatible(record.field, measure, confidence)+ ))return values@@ -924,7 +963,11 @@ class _PolicyMixin:if figure.percent:# A percent is a ratio; no price or volume may answer it.direct: list[float] = []- scaled = [] if figure.column else self._metric_pool(symbol)+ scaled = (+ []+ if figure.column+ else self._metric_pool(symbol, tail_risk=figure.tail_risk)+ )elif figure.column:direct, scaled = prices, []elif figure.currency:@@ -932,6 +975,13 @@ class _PolicyMixin:else:direct, scaled = prices + self._row_pool(symbol), self._metric_pool(symbol)if not direct and not scaled:+ detail = "no matching tool evidence"+ if figure.tail_risk is not None:+ measure, confidence = figure.tail_risk+ detail = "no {}{} evidence".format(+ measure.upper(),+ f" {confidence}%" if confidence is not None else "",+ )return [self._figure_issue("numeric_claim_unavailable",@@ -939,8 +989,8 @@ class _PolicyMixin:"observed",symbol,"no_evidence",- "is declared observed but this session holds no matching tool "- "evidence to check it against",+ f"is declared observed but this session holds {detail} "+ "to check it against",field=figure.column,date=figure.date,)diff --git a/agent/tests/test_grounding_role_hardening.py b/agent/tests/test_grounding_role_hardening.pyindex f1e44e3e0..8fb29ae78 100644--- a/agent/tests/test_grounding_role_hardening.py+++ b/agent/tests/test_grounding_role_hardening.py@@ -544,6 +544,75 @@ def test_a_tail_risk_figure_a_tool_returned_is_grounded(tmp_path: Path) -> None:assert invented.valid is False+ ("leaf", "identity"),+ [+ ("var", ("var", None)),+ ("var_95", ("var", 95)),+ ("strategy_var_99", ("var", 99)),+ ("cvar", ("es", None)),+ ("cvar_95", ("es", 95)),+ ("portfolio_cvar_95", ("es", 95)),+ ("es_99", ("es", 99)),+ ("expected_shortfall", ("es", None)),+ ("sales_es", None),+ ("var_2", None),+ ("sharpe", None),+ ],+)+def test_a_tail_risk_leaf_reports_measure_and_confidence(leaf: str, identity) -> None:+ from src.agent.grounding.evidence import _tail_risk_identity_for_path++ assert _tail_risk_identity_for_path(leaf) == identity+++def test_a_confident_var_claim_only_matches_that_confidence(tmp_path: Path) -> None:+ var = ("quantlib_call", {"action": "call", "function": "var"}, {"ok": True, "result": {"var_95": -0.0234}}, "q1")++ same = _ledger(tmp_path, MARKET_A, var).validate_final_answer(HDR + " VaR 95%: 2.34%。")+ other_level = _ledger(tmp_path / "b", MARKET_A, var).validate_final_answer(HDR + " VaR 99%: 2.34%。")+ other_measure = _ledger(tmp_path / "c", MARKET_A, var).validate_final_answer(HDR + " ES 95%: 2.34%。")++ assert same.valid is True, same.issues+ assert _reasons(other_level) == ["no_evidence"]+ assert _reasons(other_measure) == ["no_evidence"]+ # The confidence figure itself is part of the frame, never a measurement.+ assert all(issue["value"] != "95%" for result in (same, other_level) for issue in result.issues)+++def test_an_es_claim_matches_es_evidence_at_its_own_confidence(tmp_path: Path) -> None:+ cvar = ("quantlib_call", {"action": "call", "function": "cvar"}, {"ok": True, "result": {"cvar_99": -0.0311}}, "q2")++ same = _ledger(tmp_path, MARKET_A, cvar).validate_final_answer(HDR + " CVaR 99%: 3.11%。")+ alias = _ledger(tmp_path / "b", MARKET_A, cvar).validate_final_answer(HDR + " ES 99% 为 3.11%。")+ other_level = _ledger(tmp_path / "c", MARKET_A, cvar).validate_final_answer(HDR + " CVaR 95%: 3.11%。")+ var_claim = _ledger(tmp_path / "d", MARKET_A, cvar).validate_final_answer(HDR + " 单日 VaR 为 3.11%。")++ assert same.valid is True, same.issues+ assert alias.valid is True, alias.issues+ assert _reasons(other_level) == ["no_evidence"]+ assert var_claim.valid is False+++def test_a_tail_risk_frame_stops_at_the_clause_end(tmp_path: Path) -> None:+ risk = (+ "portfolio_risk_xray",+ {},+ {"ok": True, "var_95": -0.0157, "max_drawdown": -0.05132},+ "risk",+ )++ both = _ledger(tmp_path, risk, message="Analiza el riesgo de la cartera").validate_final_answer(+ "VaR 95%: 1,57%. Drawdown máximo −5,132%." + _block("95% | count | nivel de confianza")+ )+ wrong_drawdown = _ledger(tmp_path / "b", risk, message="Analiza el riesgo de la cartera").validate_final_answer(+ "VaR 95%: 1,57%. Drawdown máximo −3,9%." + _block("95% | count | nivel de confianza")+ )++ assert both.valid is True, both.issues+ assert wrong_drawdown.valid is False++# ---------------------------------------------------------------------------# B11 — what a rejected figure is pointed at# ---------------------------------------------------------------------------