"Vibe-Trading: Your Personal Trading Agent"
Python ai-agentalgorithmic-tradingbacktestingfintechllmmcpmulti-agentpythonquantitative-financetrading
fix(grounding): gate analysis metrics on a completed analysis result (#1336)
cgycorey307cc0ca0ddd
diff --git a/agent/src/agent/grounding.py b/agent/src/agent/grounding.pyindex 96add7c82..eefa00c7e 100644--- a/agent/src/agent/grounding.py+++ b/agent/src/agent/grounding.py@@ -42,6 +42,13 @@ from src.agent.resolution_context import (GROUNDING_ARTIFACT = "grounding_evidence.json"_RESOLVER_TOOL = "search_symbol"++# Tools whose successful completion grounds backtest/analysis metric claims+# (#1336). A deduplicated ("skipped") call also counts: the loop only skips a+# call that already completed successfully, and its result lives in the run.+_ANALYSIS_TOOLS = frozenset(+ {"backtest", "factor_analysis", "run_shadow_backtest", "quantlib_call"}+)_PRIVATE_COMPANY_SKILL_NAMES = {"private-company","private-company-analysis",@@ -231,6 +238,31 @@ _PRICE_CONTEXT_RE = re.compile(r"现价|报价|价格|价位)",re.IGNORECASE,)+_ANALYSIS_METRIC_RE = re.compile(+ r"(?:\breturn vol(?:atility)?\b|\bmax(?: |\.)?drawdown\b|\bmaxdd\b|"+ r"\bsharpe(?: ratio)?\b|\bwin rate\b|\bhit rate\b|"+ r"\bprob(?:ability)?\.?\s+of\b|\bvolatility\b|\bdrawdown\b|"+ r"\bannualiz\w*\b|\bwindow(?:s)?\b|\bregime(?:s)?\b|"+ r"夏普|回撤|波动率|胜率|命中率|概率|年化|回测|窗口)",+ re.IGNORECASE,+)+# Measurement-shaped numbers for analysis claims: keeps the % sign and sign+# (unlike _numbers_without_dates_or_percent, which drops percentages on+# purpose). A bare integer is NOT a measurement — "252 个交易日年化" is the+# standard convention, not a claim about this run's analysis (#1032-style+# definitional prose must stay untouched). Dates can still look like+# measurements, so date masks run first.+_MEASURE_NUMBER_RE = re.compile(+ r"[-+]?(?:\d{1,3}(?:,\d{3})+|\d+)(?:\.\d+)\s*[%%]?" # decimal: 1.21, -8.1%+ r"|[-+]?(?:\d{1,3}(?:,\d{3})+|\d+)\s*[%%]" # integer percent: 55%, 70%+)+# A forward-looking frame means the figure is a forecast, not a claim that a+# backtest/analysis measured it — the measurement gate only polices "measured+# facts" (#1336), forecasts stay under the "analysis, not advice" prompt rule.+_FORECAST_FRAME_RE = re.compile(+ r"(?:\b(?:forecast|expected|projected|predicted)\b|预计|预期|预测|估计|展望)",+ re.IGNORECASE,+)_DERIVATION_RE = re.compile(r"(?:\bderived\b|\bcalculated\b|\bformula\b|\bbased on\b|计算|推导|公式|基于)",re.IGNORECASE,@@ -1052,6 +1084,7 @@ class GroundingLedger:self._identities: dict[str, IdentityRecord] = {}self._evidence: list[EvidenceRecord] = []self._tool_failures: list[dict[str, Any]] = []+ self._analysis_completed: list[dict[str, Any]] = []self._validations: list[dict[str, Any]] = []self._recovery_rounds = 0self._symbol_resolution_attempts = 0@@ -1342,6 +1375,10 @@ class GroundingLedger:returnself._track_session_symbols(arguments, result)+ if tool_name in _ANALYSIS_TOOLS:+ self._analysis_completed.append(+ {"call_id": call_id, "tool": tool_name, "recorded_at": _utc_now()}+ )if tool_name == _RESOLVER_TOOL:self._ingest_resolution(arguments, payload, call_id)elif tool_name == "get_market_data":@@ -1365,6 +1402,7 @@ class GroundingLedger:issues.extend(self._validate_identity(content))issues.extend(self._validate_unsourced_symbols(content))issues.extend(self._validate_price_claims(content))+ issues.extend(self._validate_analysis_claims(content))result = ValidationResult(valid=not issues, issues=issues)self._validations.append({@@ -1393,7 +1431,7 @@ class GroundingLedger:for issue in validation.issues:code = issue.get("code")value = issue.get("value")- if code in {"numeric_claim_conflict", "numeric_claim_unavailable", "unsourced_symbol_figures"} and value is not None:+ if code in {"numeric_claim_conflict", "numeric_claim_unavailable", "unsourced_symbol_figures", "analysis_claim_unavailable"} and value is not None:symbol = issue.get("symbol") or ""label = f"{value:g}" if isinstance(value, (int, float)) else str(value)banned.append(f"{label} ({symbol})" if symbol else label)@@ -2304,6 +2342,105 @@ class GroundingLedger:)return issues+ def _validate_analysis_claims(self, content: str) -> list[dict[str, Any]]:+ """Reject backtest/analysis metrics with no supporting tool result.++ #1336: after failed or deduplicated market-data calls the model may+ still present return-volatility / drawdown / probability figures as+ measured facts. Two support paths make a metric figure legitimate:++ * a completed analysis tool (``_analysis_completed``) — the backtest+ family, whose metric numbers live in run-dir files and therefore+ never enter ``_evidence``; and+ * value-level match against observed evidence — any successful tool's+ numeric output (e.g. ``portfolio_risk_xray``) is flattened into+ ``_evidence``, so a figure the run actually observed is accepted.++ A figure with neither origin has no source but model memory. This is+ the same shape as ``_compare_price_claim``: value in the observed+ corpus → pass; value absent → reject. Note that a hand-maintained+ analysis-tool list is deliberately avoided (see the test that replaced+ the nine-tool price list with a uniqueness check).++ Args:+ content: Candidate assistant answer.++ Returns:+ One issue per metric-bearing clause or table row.+ """+ if self._analysis_completed:+ return []+ issues: list[dict[str, Any]] = []+ lines = content.splitlines()+ table_mode = any(+ line.count("|") >= 2 and _ANALYSIS_METRIC_RE.search(line)+ for line in lines+ )+ for line in lines:+ is_table_line = table_mode and line.count("|") >= 2+ # A pipe table whose header names metrics: every numeric cell row+ # is a metric claim, even though the marker lives on the header+ # line rather than the data row.+ segments = [line] if is_table_line else _split_clauses(line)+ for segment in segments:+ masked = _LOCALIZED_DATE_RE.sub(" ", segment)+ masked = _DATE_RE.sub(" ", masked)+ masked = _SHORT_DATE_RE.sub(" ", masked)+ masked = _DASH_DATE_RE.sub(" ", masked)+ values = [+ match.group(0).replace(" ", "").replace(",", "")+ for match in _MEASURE_NUMBER_RE.finditer(masked)+ ]+ if not values:+ continue+ if not is_table_line and not _ANALYSIS_METRIC_RE.search(segment):+ continue+ if _FORECAST_FRAME_RE.search(segment):+ continue+ unsupported = [+ value for value in values if not self._analysis_value_observed(value)+ ]+ if not unsupported:+ continue+ issues.append(+ {+ "code": "analysis_claim_unavailable",+ "claim": segment.strip()[:200],+ "value": unsupported[0],+ "message": (+ "No backtest or analysis tool completed successfully "+ "in this session, yet the answer states analysis "+ "figures. Mark the analysis as incomplete and omit "+ "these figures."+ ),+ }+ )+ return issues++ def _analysis_value_observed(self, raw: str) -> bool:+ """Return True when a claim measurement matches observed evidence.++ Tools disagree on scale: ``compute_risk_xray`` returns fractions+ (annualized_vol 0.182, max_drawdown -0.094) while answers quote+ percents (18.2%, -9.4%). Try both the value and its percent scaling+ so an observed 0.182 grounds an 18.2% claim and vice versa.+ """+ try:+ value = float(raw.replace("%", "").replace("%", "").replace(",", ""))+ except ValueError:+ return True+ observed = [+ float(record.value)+ for record in self._evidence+ if record.value is not None and record.status == "observed"+ ]+ candidates = {value, value / 100.0}+ return any(+ abs(candidate - item) <= max(abs(item) * 0.005, 1e-9)+ for candidate in candidates+ for item in observed+ )+def _validate_price_claims(self, content: str) -> list[dict[str, Any]]:"""Check Markdown OHLC tables and price prose against observed records.diff --git a/agent/tests/test_agent_grounding.py b/agent/tests/test_agent_grounding.pyindex 07888d7de..2a2005430 100644--- a/agent/tests/test_agent_grounding.py+++ b/agent/tests/test_agent_grounding.py@@ -2788,3 +2788,176 @@ def test_fx_pair_resolution_authorizes_market_data_consumer(tmp_path: Path) -> Nassert ledger.identity_status == "locked"assert ledger.authorized_symbols == {"GBPUSD=X"}assert authorization.allowed is True+++def test_backtest_metrics_rejected_when_analysis_tool_failed(+ tmp_path: Path,+) -> None:+ """#1336: failed analysis tools cannot ground backtest-style metrics."""+ ledger = GroundingLedger(+ run_dir=tmp_path,+ user_message="比较几支标的并回测不同市场状态",+ )+ ledger.ingest_tool_result(+ tool_name="backtest",+ arguments={"run_dir": "runs/compare"},+ result=(+ '{"status": "error", "error": "market data unavailable after dedup"}'+ ),+ call_id="bt-failed",+ success=False,+ )++ bad = ledger.validate_final_answer(+ "| 策略 | Return vol | MaxDD | Prob. of hitting target |\n"+ "|---|---:|---:|---:|\n"+ "| M1 | 12.4% | -8.1% | 55% |"+ )++ assert bad.valid is False, bad.issues+ assert any(+ issue["code"] == "analysis_claim_unavailable" for issue in bad.issues+ )+++def test_backtest_metrics_rejected_when_no_analysis_result(+ tmp_path: Path,+) -> None:+ """#1336: without any completed analysis, metric prose is unsupported."""+ ledger = GroundingLedger(+ run_dir=tmp_path,+ user_message="比较几个标的的历史行情",+ )++ bad = ledger.validate_final_answer(+ "历史回测显示策略年化波动率约 18.2%,最大回撤 -9.4%,"+ "夏普比率 1.21,命中目标概率 58%。"+ )++ assert bad.valid is False, bad.issues+ assert any(+ issue["code"] == "analysis_claim_unavailable" for issue in bad.issues+ )+++def test_backtest_metrics_accepted_after_successful_backtest(+ tmp_path: Path,+) -> None:+ """#1336: a genuinely completed backtest grounds the same figures."""+ ledger = GroundingLedger(+ run_dir=tmp_path,+ user_message="比较几支标的并回测不同市场状态",+ )+ ledger.ingest_tool_result(+ tool_name="backtest",+ arguments={"run_dir": "runs/compare"},+ result=(+ '{"exit_code": 0, "stdout": "ok", "stderr": "", "artifacts": '+ '["metrics.csv"], "run_dir": "runs/compare"}'+ ),+ call_id="bt-ok",+ success=True,+ )++ good = ledger.validate_final_answer(+ "| 策略 | Return vol | MaxDD | Prob. of hitting target |\n"+ "|---|---:|---:|---:|\n"+ "| M1 | 12.4% | -8.1% | 55% |"+ )++ assert good.valid is True, good.issues+++def test_analysis_mention_without_figures_is_allowed(tmp_path: Path) -> None:+ """#1336: refusal prose naming the gap is not a quantitative claim."""+ ledger = GroundingLedger(+ run_dir=tmp_path,+ user_message="回测这几个标的",+ )++ good = ledger.validate_final_answer(+ "回测未能完成(行情数据不可用),因此无法给出波动率或回撤数据。"+ )++ assert good.valid is True, good.issues+++def test_metrics_from_successful_numeric_tool_are_allowed(+ tmp_path: Path,+) -> None:+ """A successful generic analysis result grounds its returned metrics.++ The real tool returns fractions (annualized_vol 0.182, max_drawdown+ -0.094) while answers quote percents — the unit scaling must match.+ """+ ledger = GroundingLedger(+ run_dir=tmp_path,+ user_message="分析组合风险",+ )+ ledger.ingest_tool_result(+ tool_name="portfolio_risk_xray",+ arguments={"symbols": ["AAPL.US"]},+ result=json.dumps(+ {+ "status": "ok",+ "data": {+ "volatility": {"annualized_vol": 0.182},+ "drawdown": {"max_drawdown": -0.094},+ "sharpe": 1.21,+ },+ }+ ),+ call_id="risk-ok",+ success=True,+ )++ good = ledger.validate_final_answer(+ "组合年化波动率 18.2%,最大回撤 -9.4%,夏普比率 1.21。"+ )++ assert good.valid is True, good.issues+++def test_partially_unsupported_analysis_metrics_are_rejected(+ tmp_path: Path,+) -> None:+ """One observed metric cannot launder another invented metric."""+ ledger = GroundingLedger(+ run_dir=tmp_path,+ user_message="分析组合风险",+ )+ ledger.ingest_tool_result(+ tool_name="portfolio_risk_xray",+ arguments={"symbols": ["AAPL.US"]},+ result=json.dumps(+ {+ "status": "ok",+ "data": {"volatility": {"annualized_vol": 0.182}},+ }+ ),+ call_id="risk-partial",+ success=True,+ )++ bad = ledger.validate_final_answer(+ "组合年化波动率 18.2%,最大回撤 -9.4%。"+ )++ assert bad.valid is False, bad.issues+ assert any(+ issue["code"] == "analysis_claim_unavailable" for issue in bad.issues+ )+++def test_forecast_probability_is_not_a_measured_claim(tmp_path: Path) -> None:+ """#1336 gates measured facts, not forward-looking forecasts."""+ ledger = GroundingLedger(+ run_dir=tmp_path,+ user_message="明天市场会怎么样",+ )++ good = ledger.validate_final_answer(+ "预计明日上涨概率 70%,波动率可能放大。"+ )++ assert good.valid is True, good.issues