Skip to content

"Vibe-Trading: Your Personal Trading Agent"

fix(grounding): field refs resolve per symbol and by trailing path, and say when they are ambiguous (#1444)

Haozhe Wu67af91f3c8e8
diff --git a/agent/src/agent/context.py b/agent/src/agent/context.py
index 523fbe7d3..fe3da4507 100644
--- a/agent/src/agent/context.py
+++ b/agent/src/agent/context.py
@@ -193,10 +193,10 @@ Decide which workflow to use based on the request:
`value | role | note | ref`. Roles:
`observed` — a tool value that is not a price or volume of the symbol, e.g. a
PE ratio (`ref`: the tool name such as `get_fundamentals`, or its call id). For
- structured analysis metrics whose field identity matters (for example VaR/ES
- confidence), use the exact evidence field path as `ref` (for example
- `var.var_95`). If the same field was produced by more than one call, qualify
- it as `call_id::field` (for example `q1::var.var_95`) instead of pooling calls;
+ a metric whose identity matters (VaR vs ES, 95% vs 99%), use the result field
+ as `ref`: `data.tail_risk.var_95` or just `var_95` from `portfolio_risk_xray`,
+ `historical_var` from `quantlib_call`. When more than one call returned that
+ field (historical_var at 95% and at 99%), name the call: `q1::historical_var`;
`derived` — arithmetic on observed values (`note`: the formula; every number
added or subtracted must itself be an observed value);
`proposed` — a price level you suggest, such as an entry, stop or target: inside
diff --git a/agent/src/agent/grounding/policies.py b/agent/src/agent/grounding/policies.py
index 07e54b9dd..bf1b801c8 100644
--- a/agent/src/agent/grounding/policies.py
+++ b/agent/src/agent/grounding/policies.py
@@ -682,45 +682,18 @@ class _PolicyMixin:
call_id, field = (part.strip() for part in key.split("::", 1))
if not call_id or not field:
return [], []
- records = [
- record
- for record in self._evidence
- if record.call_id == call_id
- and record.field == field
- and record.status == "observed"
- and record.value is not None
- ]
- metrics = [
- float(entry["value"])
- for entry in self._analysis_metrics
- if entry.get("call_id") == call_id
- and entry.get("field") == field
- and entry.get("value") is not None
- ]
+ records, entries = self._field_sources(field, symbol)
+ records = [record for record in records if record.call_id == call_id]
+ metrics = [float(entry["value"]) for entry in entries if entry.get("call_id") == call_id]
else:
- field_records = [
- record
- for record in self._evidence
- if record.field == key
- and record.status == "observed"
- and record.value is not None
- ]
- field_metric_entries = [
- entry
- for entry in self._analysis_metrics
- if entry.get("field") == key and entry.get("value") is not None
- ]
- field_call_ids = {record.call_id for record in field_records if record.call_id} | {
- str(entry.get("call_id")) for entry in field_metric_entries if entry.get("call_id")
- }
- field_scoped = bool(field_records or field_metric_entries)
- if field_scoped and len(field_call_ids) > 1:
- # A field-only ref that occurs in multiple calls is ambiguous.
- # Fail closed rather than pooling values from different runs.
- return [], []
- if field_scoped:
- records = field_records
- metrics = [float(entry["value"]) for entry in field_metric_entries]
+ records, entries = self._field_sources(key, symbol)
+ if records or entries:
+ if self._ambiguous_field_sources(key, symbol):
+ # Calls disagree on this field, so the ref cannot say
+ # which value it quotes (VaR at 95% vs 99%). Fail closed
+ # rather than pool them; the issue names the calls.
+ return [], []
+ metrics = [float(entry["value"]) for entry in entries]
else:
records = [
record
@@ -760,6 +733,57 @@ class _PolicyMixin:
metrics = []
return records, metrics
+ def _field_sources(
+ self, field: str, symbol: str | None
+ ) -> tuple[list[EvidenceRecord], list[dict[str, Any]]]:
+ """Observed values of the evidence field ``field`` names, for ``symbol``.
+
+ ``field`` is a full path (``data.tail_risk.var_95``) or its trailing
+ part (``var_95``, ``tail_risk.var_95``): a short ref used to fall
+ through to the whole evidence pool, where a VaR 99% claim quoting the
+ 95% value found its match (#1444 review). Another symbol's records are
+ dropped here, before any count of calls, so two instruments' closes do
+ not make ``close`` ambiguous.
+ """
+ def named(path: Any) -> bool:
+ return isinstance(path, str) and (path == field or path.endswith("." + field))
+
+ records = [
+ record
+ for record in self._evidence
+ if named(record.field)
+ and record.status == "observed"
+ and record.value is not None
+ and (not symbol or not record.symbol or record.symbol == symbol)
+ ]
+ entries = [
+ entry
+ for entry in self._analysis_metrics
+ if named(entry.get("field")) and entry.get("value") is not None
+ ]
+ return records, entries
+
+ def _ambiguous_field_sources(self, field: str, symbol: str | None) -> list[str]:
+ """The ``call::path`` sources a field-only ref cannot choose between.
+
+ Ambiguous means more than one (call, path) source AND more than one
+ value among them: two runs of one call returning the same number leave
+ nothing to choose. Each source is written as the ref that names it.
+
+ Returns:
+ Sorted ``call::path`` refs, or an empty list when the ref is exact.
+ """
+ if not field or "::" in field:
+ return []
+ records, entries = self._field_sources(field, symbol)
+ sources = {(record.call_id, record.field, float(record.value)) for record in records}
+ sources |= {
+ (str(entry.get("call_id")), entry.get("field"), float(entry["value"])) for entry in entries
+ }
+ if len({(call, path) for call, path, _ in sources}) < 2 or len({value for *_, value in sources}) < 2:
+ return []
+ return sorted({f"{call}::{path}" for call, path, _ in sources})
+
def _price_pool(
self,
symbol: str | None,
@@ -1009,6 +1033,21 @@ class _PolicyMixin:
money = figure.currency and not figure.percent
if self._matches_evidence(figure, values, [] if money else values):
return []
+ ambiguous = self._ambiguous_field_sources(declaration.ref, symbol)
+ if ambiguous:
+ return [
+ self._figure_issue(
+ "numeric_claim_conflict",
+ figure,
+ "observed",
+ symbol,
+ "ambiguous_field_ref",
+ f"is declared observed from {declaration.ref}, which names "
+ f"{', '.join(ambiguous)}, and they hold different values",
+ source_tool_call_ids=[declaration.ref],
+ ambiguous_sources=ambiguous,
+ )
+ ]
return [
self._figure_issue(
"numeric_claim_conflict",
@@ -1037,7 +1076,6 @@ class _PolicyMixin:
else:
direct, scaled = prices + self._row_pool(symbol), self._metric_pool(symbol)
if not direct and not scaled:
- detail = "no matching tool evidence"
return [
self._figure_issue(
"numeric_claim_unavailable",
@@ -1045,8 +1083,8 @@ class _PolicyMixin:
"observed",
symbol,
"no_evidence",
- f"is declared observed but this session holds {detail} "
- "to check it against",
+ "is declared observed but this session holds no matching tool "
+ "evidence to check it against",
field=figure.column,
date=figure.date,
)
diff --git a/agent/src/agent/grounding/release.py b/agent/src/agent/grounding/release.py
index 6f2d40e0d..f9eaa5c27 100644
--- a/agent/src/agent/grounding/release.py
+++ b/agent/src/agent/grounding/release.py
@@ -73,6 +73,7 @@ _CORRECTION_REASONS = {
"no_evidence": "this session holds no evidence of that kind to check it against",
"value_mismatch": "the observed evidence is {range}",
"not_in_referenced_call": "call {ref} returned no such value",
+ "ambiguous_field_ref": "{ref} names {sources}, which hold different values; use the one quoted as the ref",
"no_formula": "its note states no arithmetic",
"formula_not_evaluable": "its note is not an arithmetic expression over two or more operands",
"formula_not_anchored": "no operand of its note is a value this session observed",
@@ -116,6 +117,7 @@ def _correction_line(issue: dict[str, Any]) -> str:
evidence = template.format(
range=span,
ref=issue.get("source_tool_call_ids", [""])[0] if issue.get("source_tool_call_ids") else "",
+ sources=", ".join(str(source) for source in issue.get("ambiguous_sources") or []),
result=result if result else "a different value",
)
nearest = issue.get("observed_nearest") or []
diff --git a/agent/tests/test_grounding_role_hardening.py b/agent/tests/test_grounding_role_hardening.py
index 72e2c5c6e..dab72c913 100644
--- a/agent/tests/test_grounding_role_hardening.py
+++ b/agent/tests/test_grounding_role_hardening.py
@@ -544,95 +544,118 @@ def test_a_tail_risk_figure_a_tool_returned_is_grounded(tmp_path: Path) -> None:
assert invented.valid is False
-def test_an_exact_metric_field_ref_is_tighter_than_a_call_ref(tmp_path: Path) -> None:
- risk = (
- "quantlib_call",
- {"action": "call", "function": "var"},
- {"ok": True, "result": {"var_95": -0.0157, "var_99": -0.0263}},
- "q1",
- )
- prose = HDR + " VaR 95%: 1.57%。"
- confidence = "95% | count | confidence"
-
- exact = _ledger(tmp_path / "exact", MARKET_A, risk).validate_final_answer(
- prose + _block(ROW, confidence, "1.57% | observed | VaR 95% | var.var_95")
- )
- wrong_field = _ledger(tmp_path / "wrong", MARKET_A, risk).validate_final_answer(
- prose + _block(ROW, confidence, "1.57% | observed | VaR 95% | var.var_99")
- )
- call_scope = _ledger(tmp_path / "call", MARKET_A, risk).validate_final_answer(
- prose + _block(ROW, confidence, "1.57% | observed | VaR 95% | q1")
- )
+#: portfolio_risk_xray's tail-risk block, keys as backtest/risk_xray.py writes them.
+XRAY = (
+ "portfolio_risk_xray",
+ {"symbols": [A]},
+ {
+ "status": "ok",
+ "data": {
+ "tail_risk": {
+ "var_95": 0.0157,
+ "expected_shortfall_95": 0.0211,
+ "var_99": 0.0263,
+ "expected_shortfall_99": 0.0342,
+ }
+ },
+ },
+ "x1",
+)
- assert exact.valid is True, exact.issues
- assert _reasons(wrong_field) == ["not_in_referenced_call"]
- assert call_scope.valid is True, call_scope.issues
+def _historical_var(confidence: float, call_id: str) -> tuple[tuple[str, dict[str, Any], str, str], float]:
+ """A real quantlib_call historical_var envelope and the VaR it returned."""
+ from src.tools.quantlib_tool import QuantlibCallTool
-def test_es_and_var_fields_do_not_borrow_each_others_values(tmp_path: Path) -> None:
- risk = (
- "quantlib_call",
- {"action": "call", "function": "var"},
- {"ok": True, "result": {"var_95": -0.0157, "cvar_95": -0.0211}},
- "q1",
- )
- prose = HDR + " ES 95%: 2.11%。"
- block_head = (ROW, "95% | count | confidence")
+ returns = [round(0.0005 + 0.012 * ((i * 37) % 101 - 50) / 50, 5) for i in range(250)]
+ arguments = {"action": "call", "module": "risk", "function": "historical_var", "kwargs": {"returns": returns, "confidence": confidence}}
+ payload = QuantlibCallTool().execute(**arguments)
+ return ("quantlib_call", arguments, payload, call_id), float(json.loads(payload)["result"])
- es = _ledger(tmp_path / "es", MARKET_A, risk).validate_final_answer(
- prose + _block(*block_head, "2.11% | observed | ES 95% | var.cvar_95")
- )
- wrong_measure = _ledger(tmp_path / "var", MARKET_A, risk).validate_final_answer(
- prose + _block(*block_head, "2.11% | observed | ES 95% | var.var_95")
- )
- assert es.valid is True, es.issues
- assert _reasons(wrong_measure) == ["not_in_referenced_call"]
+def _pct(value: float) -> str:
+ return f"{abs(value) * 100:.2f}%"
-def test_repeated_field_refs_require_call_qualification(tmp_path: Path) -> None:
- first = (
- "quantlib_call",
- {"action": "call", "function": "var"},
- {"ok": True, "result": {"var_95": -0.0157}},
- "q1",
- )
- second = (
- "quantlib_call",
- {"action": "call", "function": "var"},
- {"ok": True, "result": {"var_95": -0.0999}},
- "q2",
[email protected]("ref", ["data.tail_risk.var_95", "tail_risk.var_95", "var_95", "x1"])
+def test_a_tail_risk_ref_grounds_its_own_value(tmp_path: Path, ref: str) -> None:
+ result = _ledger(tmp_path, MARKET_A, XRAY).validate_final_answer(
+ HDR + " VaR 95%: 1.57%。" + _block(ROW, "95% | count | confidence", f"1.57% | observed | VaR 95% | {ref}")
)
- prose = HDR + " VaR 95%: 1.57%。"
- confidence = "95% | count | confidence"
+ assert result.valid is True, result.issues
+
- ambiguous = _ledger(
- tmp_path / "ambiguous", MARKET_A, first, second
- ).validate_final_answer(
- prose + _block(ROW, confidence, "1.57% | observed | VaR 95% | var.var_95")
+ ("claim", "ref"),
+ [
+ ("VaR 99%: 1.57%", "data.tail_risk.var_99"),
+ # A short ref used to name no field and fall through to the whole pool,
+ # where the 95% value answered a 99% claim (#1444 review, R18).
+ ("VaR 99%: 1.57%", "var_99"),
+ ("ES 95%: 1.57%", "expected_shortfall_95"),
+ ],
+)
+def test_a_field_ref_does_not_borrow_another_fields_value(tmp_path: Path, claim: str, ref: str) -> None:
+ confidence = "99% | count | confidence" if "99%" in claim else "95% | count | confidence"
+ result = _ledger(tmp_path, MARKET_A, XRAY).validate_final_answer(
+ HDR + f" {claim}。" + _block(ROW, confidence, f"1.57% | observed | {claim} | {ref}")
)
- q1 = _ledger(tmp_path / "q1", MARKET_A, first, second).validate_final_answer(
- prose
- + _block(
- ROW,
- confidence,
- "1.57% | observed | VaR 95% | q1::var.var_95",
+ assert _reasons(result) == ["not_in_referenced_call"]
+
+
+def test_a_field_two_calls_returned_differently_needs_its_call(tmp_path: Path) -> None:
+ """historical_var at 95% and at 99% is one field in two calls: the bare
+ field ref cannot say which it quotes, and the correction names both."""
+ q1, var_95 = _historical_var(0.95, "q1")
+ q2, var_99 = _historical_var(0.99, "q2")
+ assert var_95 != var_99
+ prose = HDR + f" VaR 95%: {_pct(var_95)}。"
+
+ def answer(ref: str):
+ return _ledger(tmp_path / ref.replace(":", "_"), MARKET_A, q1, q2).validate_final_answer(
+ prose + _block(ROW, "95% | count | confidence", f"{_pct(var_95)} | observed | VaR 95% | {ref}")
)
+
+ ambiguous = answer("historical_var")
+ assert _reasons(ambiguous) == ["ambiguous_field_ref"]
+ assert ambiguous.issues[0]["ambiguous_sources"] == ["q1::historical_var", "q2::historical_var"]
+ assert answer("q1::historical_var").valid is True
+ assert _reasons(answer("q2::historical_var")) == ["not_in_referenced_call"]
+
+
+def test_a_field_two_calls_returned_identically_is_not_ambiguous(tmp_path: Path) -> None:
+ q1, var_95 = _historical_var(0.95, "q1")
+ q2, same = _historical_var(0.95, "q2")
+ assert same == var_95
+ result = _ledger(tmp_path, MARKET_A, q1, q2).validate_final_answer(
+ HDR + f" VaR 95%: {_pct(var_95)}。"
+ + _block(ROW, "95% | count | confidence", f"{_pct(var_95)} | observed | VaR 95% | historical_var")
)
- wrong_call = _ledger(
- tmp_path / "wrong", MARKET_A, first, second
- ).validate_final_answer(
- prose
- + _block(
- ROW,
- confidence,
- "1.57% | observed | VaR 95% | q2::var.var_95",
- )
+ assert result.valid is True, result.issues
+
+
+def test_another_symbols_field_does_not_make_a_ref_ambiguous(tmp_path: Path) -> None:
+ """Two instruments' bars both carry ``close``; the figure's own symbol
+ decides before any count of calls (#1444 review, P9)."""
+ result = _ledger(tmp_path, MARKET_A, MARKET_B).validate_final_answer(
+ TWO + f"{A} 最新收盘 0.666 元。" + _block("0.666 | observed | close | close")
)
+ assert result.valid is True, result.issues
- assert _reasons(ambiguous) == ["not_in_referenced_call"]
- assert q1.valid is True, q1.issues
- assert _reasons(wrong_call) == ["not_in_referenced_call"]
+
+def test_the_ambiguity_correction_names_the_refs_to_use() -> None:
+ from src.agent.grounding.release import _correction_line
+
+ line = _correction_line(
+ {
+ "value": "1.98%",
+ "role": "observed",
+ "reason": "ambiguous_field_ref",
+ "source_tool_call_ids": ["historical_var"],
+ "ambiguous_sources": ["q1::historical_var", "q2::historical_var"],
+ }
+ )
+ assert "historical_var names q1::historical_var, q2::historical_var" in line
@pytest.mark.parametrize(
@@ -652,13 +675,7 @@ def test_repeated_field_refs_require_call_qualification(tmp_path: Path) -> None:
def test_tail_risk_prose_does_not_skip_undeclared_numbers(
tmp_path: Path, claim: str
) -> None:
- risk = (
- "quantlib_call",
- {"action": "call", "function": "var"},
- {"ok": True, "result": {"var_95": -0.0157, "var_99": -0.0263}},
- "q1",
- )
- result = _ledger(tmp_path, MARKET_A, risk).validate_final_answer(HDR + " " + claim)
+ result = _ledger(tmp_path, MARKET_A, XRAY).validate_final_answer(HDR + " " + claim)
assert result.valid is False