Skip to content

"Vibe-Trading: Your Personal Trading Agent"

fix(grounding): keep tail-risk measure and confidence through the gate

Yufeng He452c64db1139
diff --git a/agent/src/agent/grounding/evidence.py b/agent/src/agent/grounding/evidence.py
index 5c6fe8157..f003a3256 100644
--- a/agent/src/agent/grounding/evidence.py
+++ b/agent/src/agent/grounding/evidence.py
@@ -123,7 +123,11 @@ _ANALYSIS_KIND_ALIASES = {
"var_95": "tail_risk",
"var_99": "tail_risk",
"cvar": "tail_risk",
+ "cvar_95": "tail_risk",
+ "cvar_99": "tail_risk",
"es": "tail_risk",
+ "es_95": "tail_risk",
+ "es_99": "tail_risk",
"expected_shortfall": "tail_risk",
# Chinese TOOL FIELD NAMES from A-share tools, not answer prose.
"最大回撤": "drawdown",
@@ -357,17 +361,16 @@ _QUALIFIER_SUFFIXES = frozenset(
)
-def _metric_kind_for_path(path: str) -> str | None:
- """Map an evidence JSON path to an analysis metric kind.
+def _metric_alias_key_for_path(path: str) -> str | None:
+ """The alias-table key an evidence JSON path resolves to, or None.
- A metadata count leaf (:func:`_is_metadata_count_leaf`) has no kind.
+ A metadata count leaf (:func:`_is_metadata_count_leaf`) resolves to nothing.
"""
if _is_metadata_count_leaf(path):
return None
leaf = _leaf_name(path)
- kind = _ANALYSIS_KIND_ALIASES.get(leaf)
- if kind is not None:
- return kind
+ if leaf in _ANALYSIS_KIND_ALIASES:
+ return leaf
# Compound leaves ("strategy_max_drawdown"): scan tokens from the right,
# where English puts the head noun, so "return_vol" is vol, not return.
# Only the head of a compound leaf says what it measures (#1426):
@@ -383,12 +386,60 @@ def _metric_kind_for_path(path: str) -> str | None:
if len(candidate) < size:
continue
key = "_".join(candidate[-size:])
- kind = None if key in _EXACT_ONLY_ALIASES else _ANALYSIS_KIND_ALIASES.get(key)
- if kind is not None:
- return kind
+ if key in _EXACT_ONLY_ALIASES:
+ continue
+ if key in _ANALYSIS_KIND_ALIASES:
+ return key
return None
+def _metric_kind_for_path(path: str) -> str | None:
+ """Map an evidence JSON path to an analysis metric kind.
+
+ A metadata count leaf (:func:`_is_metadata_count_leaf`) has no kind.
+ """
+ key = _metric_alias_key_for_path(path)
+ return _ANALYSIS_KIND_ALIASES.get(key) if key is not None else None
+
+
+# What a tail-risk measure word reports: an ES and a VaR are different
+# measurements even when they share a confidence level and a number.
+_TAIL_RISK_MEASURES = {
+ "var": "var",
+ "cvar": "es",
+ "es": "es",
+ "expected_shortfall": "es",
+}
+
+
+def _tail_risk_identity_for_path(path: str) -> tuple[str, int | None] | None:
+ """The ``(measure, confidence)`` a tail-risk evidence path reports.
+
+ ``var_95`` is a 95% VaR: not an ES, and not a 99% anything. A numeric token
+ right after the measure head is the confidence; names without one carry
+ ``None`` and can only ground a claim that states no confidence.
+ """
+ key = _metric_alias_key_for_path(path)
+ if key is None or _ANALYSIS_KIND_ALIASES.get(key) != "tail_risk":
+ return None
+ if key in _TAIL_RISK_MEASURES:
+ # A compound leaf can still carry the confidence after the head:
+ # "portfolio_cvar_95" resolves as "cvar", with 95 stripped as a
+ # qualifier. Recover it from the leaf itself.
+ tokens = [token for token in re.split(r"[_.]", _leaf_name(path)) if token]
+ for i in range(len(tokens) - 1, -1, -1):
+ head_len = 2 if "_".join(tokens[i : i + 2]) == key else 1 if tokens[i] == key else 0
+ if head_len:
+ following = tokens[i + head_len] if i + head_len < len(tokens) else ""
+ return _TAIL_RISK_MEASURES[key], int(following) if following.isdigit() else None
+ return _TAIL_RISK_MEASURES[key], None
+ head, _, tail = key.rpartition("_")
+ measure = _TAIL_RISK_MEASURES.get(head)
+ if measure is None or not tail.isdigit():
+ return None
+ return measure, int(tail)
+
+
@dataclass(frozen=True)
class EvidenceRecord:
"""One observed, unavailable, or derived numeric evidence item."""
diff --git a/agent/src/agent/grounding/figures.py b/agent/src/agent/grounding/figures.py
index 3557e8fcf..067aef36d 100644
--- a/agent/src/agent/grounding/figures.py
+++ b/agent/src/agent/grounding/figures.py
@@ -17,7 +17,7 @@ import html
import re
import string
import unicodedata
-from dataclasses import dataclass
+from dataclasses import dataclass, replace
from decimal import Decimal, InvalidOperation
from typing import Iterable, Sequence
@@ -180,6 +180,11 @@ class Figure:
extent: tuple[int, int] | None = None
# The info string of the fenced block holding the figure, or None.
fence: str | None = None
+ # The tail-risk frame the figure sits in ("VaR 95%: 1.57%"): the measured
+ # value carries (measure, confidence); the confidence figure itself is
+ # flagged instead and never checked as a measurement.
+ tail_risk: tuple[str, int | None] | None = None
+ tail_risk_confidence: bool = False
@dataclass(frozen=True)
@@ -859,6 +864,97 @@ def _short_date_is_structural(content: str, match: re.Match[str], full_dates: Se
return any(segment <= full.start() and full.end() <= match.start() for full in full_dates)
+# A tail-risk claim's prose frame: the measure word, then an optional
+# confidence level. VaR and ES/CVaR are different measurements at any
+# confidence, and a 95% and a 99% VaR are different ones.
+_TAIL_RISK_WORD_RE = re.compile(
+ r"(?<![A-Za-z])(?:C?VaR|ES)(?![A-Za-z])|(?i:expected shortfall)|在险价值|风险价值|预期损失|条件在险价值"
+)
+
+# The confidence sits right after the measure word ("VaR 95%", "VaR (95%)",
+# "ES at 99%") or right before it ("95% VaR", "95% 置信水平下的 VaR").
+_TAIL_RISK_CONFIDENCE_AFTER_RE = re.compile(
+ r"\s*[((]?\s*(?:at|of)?\s*(\d{1,3}(?:\.\d+)?)\s*[%%]"
+)
+_TAIL_RISK_CONFIDENCE_BEFORE_RE = re.compile(
+ r"(\d{1,3}(?:\.\d+)?)\s*[%%]\s*(?:置信水平|置信度|confidence(?:\s+level)?)?\s*(?:下|中|的)?\s*[,,]?\s*$"
+)
+
+
+def _tail_risk_measure_of(word: str) -> str:
+ """The measure a tail-risk keyword reports: ES and CVaR are one family."""
+ return (
+ "es"
+ if word.casefold() == "expected shortfall" or word in ("ES", "CVaR", "预期损失", "条件在险价值")
+ else "var"
+ )
+
+
+def _tail_risk_frames(content: str, figures: list[Figure]) -> list[Figure]:
+ """Mark the tail-risk frame each percent figure sits in, when there is one.
+
+ The confidence level in "VaR 95%: 1.57%" is part of the frame, not a
+ measurement: it is flagged so the gate never checks it, while the value is
+ checked only against evidence of the same measure and confidence.
+ """
+ if not any(figure.percent for figure in figures):
+ return figures
+ lines = _lines_with_offsets(content)
+ out = list(figures)
+ for line_index, (line_text, line_start) in enumerate(lines):
+ line_figures = [
+ (index, figure)
+ for index, figure in enumerate(figures)
+ if figure.line == line_index and figure.percent
+ ]
+ if not line_figures:
+ continue
+ keywords = list(_TAIL_RISK_WORD_RE.finditer(line_text))
+ if not keywords:
+ continue
+ # A frame runs from its keyword to the next keyword or the clause end:
+ # "VaR 95%: 1,57%. Drawdown máximo −5,132%" keeps the drawdown out of
+ # the VaR frame entirely.
+ terminators = [
+ match.end()
+ for match in re.finditer(r"[。!!??;;.]", line_text)
+ ]
+ regions: list[tuple[int, int]] = []
+ for pos, keyword in enumerate(keywords):
+ end = keywords[pos + 1].start() if pos + 1 < len(keywords) else len(line_text)
+ boundary = next((mark for mark in terminators if mark > keyword.end()), end)
+ regions.append((keyword.start(), min(end, boundary)))
+ # (measure, confidence, confidence digit start in source coords, region)
+ frames: list[tuple[str, int | None, int | None, tuple[int, int]]] = []
+ for keyword, region in zip(keywords, regions):
+ measure = _tail_risk_measure_of(keyword.group(0))
+ confidence: int | None = None
+ confidence_start: int | None = None
+ match = _TAIL_RISK_CONFIDENCE_AFTER_RE.match(line_text, keyword.end())
+ if match is None:
+ match = _TAIL_RISK_CONFIDENCE_BEFORE_RE.search(line_text, 0, keyword.start())
+ if match:
+ confidence = int(float(match.group(1)))
+ confidence_start = line_start + match.start(1)
+ frames.append((measure, confidence, confidence_start, region))
+ for index, figure in line_figures:
+ if figure.start in {frame[2] for frame in frames}:
+ out[index] = replace(figure, tail_risk_confidence=True)
+ continue
+ line_pos = figure.start - line_start
+ region = next(
+ (frame[3] for frame in frames if frame[3][0] <= line_pos < frame[3][1]),
+ None,
+ )
+ if region is None:
+ continue
+ measure, confidence, _, _ = next(
+ frame for frame in frames if frame[3] == region
+ )
+ out[index] = replace(figure, tail_risk=(measure, confidence))
+ return out
+
+
def scan_figures(content: str, block: FiguresBlock) -> list[Figure]:
"""Locate and classify every number in the prose of a draft (spec §3).
@@ -959,4 +1055,4 @@ def scan_figures(content: str, block: FiguresBlock) -> list[Figure]:
),
)
)
- return figures
+ return _tail_risk_frames(content, figures)
diff --git a/agent/src/agent/grounding/policies.py b/agent/src/agent/grounding/policies.py
index 411a33f12..5daf14402 100644
--- a/agent/src/agent/grounding/policies.py
+++ b/agent/src/agent/grounding/policies.py
@@ -25,6 +25,7 @@ from src.agent.grounding.evidence import (
_is_price_kind,
_metric_kind_for_path,
_price_field_for_path,
+ _tail_risk_identity_for_path,
_timestamp_matches_claim_date,
)
from src.agent.grounding.figures import (
@@ -466,6 +467,11 @@ class _PolicyMixin:
if figure.shape not in ("measured", "bare"):
continue
declaration = block.match(figure.value, figure.percent)
+ if figure.tail_risk_confidence and declaration is None:
+ # The confidence level of a tail-risk frame ("VaR 95%: 1.57%")
+ # qualifies the measurement; it is not one itself. A declared
+ # role wins: the model said what the number is.
+ continue
symbol = self._figure_symbol(
content, figure, declaration, line_symbols, document_symbol, records
)
@@ -849,12 +855,41 @@ class _PolicyMixin:
return low * 0.9 <= value <= high * 1.1
return low >= _INTEGER_PRICE_FLOOR and low * 0.5 <= value <= high * 2.0
- def _metric_pool(self, symbol: str | None) -> list[float]:
- """Metric values from completed analysis results and metric-named leaves."""
+ @staticmethod
+ def _tail_risk_compatible(
+ field_name: str, measure: str, confidence: int | None
+ ) -> bool:
+ """Whether a tail-risk evidence field can ground a claim's identity.
+
+ The measure must agree; a confidence the claim states must be matched
+ exactly, while a claim stating none takes any evidence of its measure.
+ """
+ identity = _tail_risk_identity_for_path(field_name)
+ if identity is None or identity[0] != measure:
+ return False
+ return confidence is None or identity[1] == confidence
+
+ def _metric_pool(
+ self,
+ symbol: str | None,
+ *,
+ tail_risk: tuple[str, int | None] | None = None,
+ ) -> list[float]:
+ """Metric values from completed analysis results and metric-named leaves.
+
+ A tail-risk claim narrows the pool to evidence of its own measure and,
+ when the claim states one, its confidence: a ``var_95`` reading says
+ nothing about a "VaR 99%" or an "ES 95%" figure.
+ """
+ measure, confidence = tail_risk if tail_risk is not None else (None, None)
values = [
float(entry["value"])
for entry in self._analysis_metrics
if entry.get("value") is not None
+ and (
+ measure is None
+ or self._tail_risk_compatible(str(entry.get("field") or ""), measure, confidence)
+ )
]
values.extend(
float(record.value)
@@ -863,6 +898,10 @@ class _PolicyMixin:
and record.value is not None
and _metric_kind_for_path(record.field) is not None
and (not symbol or not record.symbol or record.symbol == symbol)
+ and (
+ measure is None
+ or self._tail_risk_compatible(record.field, measure, confidence)
+ )
)
return values
@@ -924,7 +963,11 @@ class _PolicyMixin:
if figure.percent:
# A percent is a ratio; no price or volume may answer it.
direct: list[float] = []
- scaled = [] if figure.column else self._metric_pool(symbol)
+ scaled = (
+ []
+ if figure.column
+ else self._metric_pool(symbol, tail_risk=figure.tail_risk)
+ )
elif figure.column:
direct, scaled = prices, []
elif figure.currency:
@@ -932,6 +975,13 @@ class _PolicyMixin:
else:
direct, scaled = prices + self._row_pool(symbol), self._metric_pool(symbol)
if not direct and not scaled:
+ detail = "no matching tool evidence"
+ if figure.tail_risk is not None:
+ measure, confidence = figure.tail_risk
+ detail = "no {}{} evidence".format(
+ measure.upper(),
+ f" {confidence}%" if confidence is not None else "",
+ )
return [
self._figure_issue(
"numeric_claim_unavailable",
@@ -939,8 +989,8 @@ class _PolicyMixin:
"observed",
symbol,
"no_evidence",
- "is declared observed but this session holds no matching tool "
- "evidence to check it against",
+ f"is declared observed but this session holds {detail} "
+ "to check it against",
field=figure.column,
date=figure.date,
)
diff --git a/agent/tests/test_grounding_role_hardening.py b/agent/tests/test_grounding_role_hardening.py
index f1e44e3e0..8fb29ae78 100644
--- a/agent/tests/test_grounding_role_hardening.py
+++ b/agent/tests/test_grounding_role_hardening.py
@@ -544,6 +544,75 @@ def test_a_tail_risk_figure_a_tool_returned_is_grounded(tmp_path: Path) -> None:
assert invented.valid is False
+ ("leaf", "identity"),
+ [
+ ("var", ("var", None)),
+ ("var_95", ("var", 95)),
+ ("strategy_var_99", ("var", 99)),
+ ("cvar", ("es", None)),
+ ("cvar_95", ("es", 95)),
+ ("portfolio_cvar_95", ("es", 95)),
+ ("es_99", ("es", 99)),
+ ("expected_shortfall", ("es", None)),
+ ("sales_es", None),
+ ("var_2", None),
+ ("sharpe", None),
+ ],
+)
+def test_a_tail_risk_leaf_reports_measure_and_confidence(leaf: str, identity) -> None:
+ from src.agent.grounding.evidence import _tail_risk_identity_for_path
+
+ assert _tail_risk_identity_for_path(leaf) == identity
+
+
+def test_a_confident_var_claim_only_matches_that_confidence(tmp_path: Path) -> None:
+ var = ("quantlib_call", {"action": "call", "function": "var"}, {"ok": True, "result": {"var_95": -0.0234}}, "q1")
+
+ same = _ledger(tmp_path, MARKET_A, var).validate_final_answer(HDR + " VaR 95%: 2.34%。")
+ other_level = _ledger(tmp_path / "b", MARKET_A, var).validate_final_answer(HDR + " VaR 99%: 2.34%。")
+ other_measure = _ledger(tmp_path / "c", MARKET_A, var).validate_final_answer(HDR + " ES 95%: 2.34%。")
+
+ assert same.valid is True, same.issues
+ assert _reasons(other_level) == ["no_evidence"]
+ assert _reasons(other_measure) == ["no_evidence"]
+ # The confidence figure itself is part of the frame, never a measurement.
+ assert all(issue["value"] != "95%" for result in (same, other_level) for issue in result.issues)
+
+
+def test_an_es_claim_matches_es_evidence_at_its_own_confidence(tmp_path: Path) -> None:
+ cvar = ("quantlib_call", {"action": "call", "function": "cvar"}, {"ok": True, "result": {"cvar_99": -0.0311}}, "q2")
+
+ same = _ledger(tmp_path, MARKET_A, cvar).validate_final_answer(HDR + " CVaR 99%: 3.11%。")
+ alias = _ledger(tmp_path / "b", MARKET_A, cvar).validate_final_answer(HDR + " ES 99% 为 3.11%。")
+ other_level = _ledger(tmp_path / "c", MARKET_A, cvar).validate_final_answer(HDR + " CVaR 95%: 3.11%。")
+ var_claim = _ledger(tmp_path / "d", MARKET_A, cvar).validate_final_answer(HDR + " 单日 VaR 为 3.11%。")
+
+ assert same.valid is True, same.issues
+ assert alias.valid is True, alias.issues
+ assert _reasons(other_level) == ["no_evidence"]
+ assert var_claim.valid is False
+
+
+def test_a_tail_risk_frame_stops_at_the_clause_end(tmp_path: Path) -> None:
+ risk = (
+ "portfolio_risk_xray",
+ {},
+ {"ok": True, "var_95": -0.0157, "max_drawdown": -0.05132},
+ "risk",
+ )
+
+ both = _ledger(tmp_path, risk, message="Analiza el riesgo de la cartera").validate_final_answer(
+ "VaR 95%: 1,57%. Drawdown máximo −5,132%." + _block("95% | count | nivel de confianza")
+ )
+ wrong_drawdown = _ledger(tmp_path / "b", risk, message="Analiza el riesgo de la cartera").validate_final_answer(
+ "VaR 95%: 1,57%. Drawdown máximo −3,9%." + _block("95% | count | nivel de confianza")
+ )
+
+ assert both.valid is True, both.issues
+ assert wrong_drawdown.valid is False
+
+
# ---------------------------------------------------------------------------
# B11 — what a rejected figure is pointed at
# ---------------------------------------------------------------------------