Skip to content

"Vibe-Trading: Your Personal Trading Agent"

fix(grounding): let a backtest's own output ground the report about it

Haozhe Wud2dd7006dff6
diff --git a/agent/src/agent/context.py b/agent/src/agent/context.py
index 1faa05995..c53fa1544 100644
--- a/agent/src/agent/context.py
+++ b/agent/src/agent/context.py
@@ -201,9 +201,14 @@ Decide which workflow to use based on the request:
Once this session holds more than one tail-risk measurement (a VaR and an ES,
or 95% and 99%), EVERY tail-risk figure needs that field ref — a call id or no
declaration at all cannot say which of them you are quoting, and the figure is
- sent back for correction;
+ sent back for correction.
+ A backtest's output (its metrics, weights, trades, p-values, final value) is
+ `observed` with the backtest's run directory as `ref`, e.g. `rp`, or the file
+ you read, e.g. `rp/artifacts/target_positions.csv`; two backtests are two
+ directories, so a comparison names each one (`rp::sharpe`, `ew::sharpe`);
`derived` — arithmetic on observed values (`note`: the formula; every number
- added or subtracted must itself be an observed value);
+ added or subtracted must itself be an observed value; `ref`: where the
+ operands came from, e.g. `rp, ew` for a difference between two backtests);
`proposed` — a price level you suggest, such as an entry, stop or target: inside
the observed price range, or with a formula over observed values in `note`; a
percentage is not a level, so state the price it implies;
diff --git a/agent/src/agent/grounding/evidence.py b/agent/src/agent/grounding/evidence.py
index 93294e5bb..cd1012f9c 100644
--- a/agent/src/agent/grounding/evidence.py
+++ b/agent/src/agent/grounding/evidence.py
@@ -7,6 +7,7 @@ keyed on tool field names, never on natural language.
from __future__ import annotations
import csv
+import hashlib
import json
import math
import re
@@ -15,6 +16,7 @@ from pathlib import Path
from typing import Any, Mapping
from src.agent.grounding.identity import (
+ _CANONICAL_SYMBOL_RE,
_infer_currency,
_infer_venue,
_normalize_symbol,
@@ -158,8 +160,141 @@ _METADATA_COUNT_TAILS = frozenset(
{"obs", "observations", "window", "lookback", "count", "days", "duration"}
)
-# Money-denominated row fields a currency-marked figure may quote besides a price.
-_AMOUNT_FIELDS = frozenset({"amount", "turnover", "成交额"})
+# Money-denominated fields a currency-marked figure may quote besides a price:
+# a market-data row's amount, and the backtest engine's equity and P&L leaves.
+_AMOUNT_FIELDS = frozenset(
+ {
+ "amount",
+ "turnover",
+ "成交额",
+ "final_value",
+ "initial_cash",
+ "initial_capital",
+ "pnl",
+ "total_pnl",
+ "avg_pnl",
+ "notional",
+ }
+)
+
+# The backtest engine's summary outputs, relative to its run directory: small
+# documents of scalars, recorded whole when the backtest completes. A list
+# inside them (1,000 Monte Carlo Sharpe samples, one entry per rebalance) is a
+# series and is not descended. metrics.csv/json are recorded as they always
+# were; every other file must appear, byte for byte, in the run card's
+# artifact manifest, which the engine writes after its outputs.
+_BACKTEST_SUMMARY_FILES = (
+ "run_card.json",
+ "artifacts/metrics.csv",
+ "artifacts/metrics.json",
+ "artifacts/risk_xray.json",
+ "artifacts/validation.json",
+ "artifacts/rebalance_notes.json",
+)
+
+_MANIFEST_EXEMPT = frozenset({"run_card.json", "artifacts/metrics.csv", "artifacts/metrics.json"})
+
+# Tools whose result names a file the model itself wrote. Such a file is never
+# engine output, whatever its name.
+_WRITE_TOOLS = frozenset({"write_file", "edit_file"})
+
+# Columns of a backtest table that name the row's instrument (trades.csv "code").
+_TABLE_SYMBOL_COLUMNS = frozenset({"symbol", "code", "ticker"})
+
+#: Written by the loop into the active run when it archives a backtest there;
+#: ``source_run`` names the run directory the copy came from.
+ARCHIVE_MANIFEST = ".archived_backtest.json"
+
+
+def _relative_posix(path: Path, root: Path) -> str:
+ """``path`` relative to ``root`` in POSIX form, "" for ``root`` itself."""
+ try:
+ relative = Path(path).resolve().relative_to(root).as_posix()
+ except (OSError, ValueError):
+ return Path(path).as_posix()
+ return "" if relative == "." else relative
+
+
+def _file_sha256(path: Path) -> str | None:
+ """Hex SHA-256 of a file's bytes, or None when it cannot be read."""
+ try:
+ return hashlib.sha256(Path(path).read_bytes()).hexdigest()
+ except OSError:
+ return None
+
+
+def _archive_source(root: Path) -> str | None:
+ """The run directory name the active run's archived backtest came from."""
+ try:
+ payload = json.loads((root / ARCHIVE_MANIFEST).read_text(encoding="utf-8"))
+ except (OSError, UnicodeDecodeError, json.JSONDecodeError):
+ return None
+ source = payload.get("source_run") if isinstance(payload, dict) else None
+ return str(source) if source else None
+
+
+def _run_card_manifest(directory: Path) -> dict[str, str]:
+ """``{relative path: sha256}`` from a backtest's run card, or empty."""
+ try:
+ card = json.loads((directory / "run_card.json").read_text(encoding="utf-8"))
+ except (OSError, UnicodeDecodeError, json.JSONDecodeError):
+ return {}
+ entries = card.get("artifacts") if isinstance(card, dict) else None
+ if not isinstance(entries, list):
+ return {}
+ return {
+ str(entry["path"]): str(entry["sha256"])
+ for entry in entries
+ if isinstance(entry, dict) and entry.get("path") and entry.get("sha256")
+ }
+
+
+def _summary_scalars(path: Path) -> list[tuple[str, int | float]]:
+ """``(field, value)`` for every scalar of a summary artifact; lists are not descended.
+
+ A CSV contributes its first data row by (casefolded) header, a JSON
+ document every number reachable through objects alone, under its dotted
+ path ("tail_risk.var_95").
+ """
+ try:
+ text = path.read_text(encoding="utf-8")
+ except (OSError, UnicodeDecodeError):
+ return []
+ pairs: list[tuple[str, int | float]] = []
+ if path.suffix == ".csv":
+ try:
+ rows = list(csv.reader(text.splitlines()))
+ except csv.Error:
+ return []
+ if len(rows) < 2:
+ return []
+ for name, cell in zip(rows[0], rows[1]):
+ value = _coerce_csv_number(cell)
+ if name.strip() and value is not None:
+ pairs.append((name.strip().casefold(), value))
+ return pairs
+ try:
+ data = json.loads(text)
+ except json.JSONDecodeError:
+ return []
+
+ def visit(item: Any, dotted: str) -> None:
+ if _is_number(item):
+ pairs.append((dotted, item))
+ elif isinstance(item, dict):
+ for key, child in item.items():
+ visit(child, f"{dotted}.{key}" if dotted else str(key))
+
+ visit(data, "")
+ return pairs
+
+
+def _cell_symbol(text: str) -> str | None:
+ """The canonical symbol a table cell or header consists of, or None."""
+ stripped = (text or "").strip()
+ if not stripped or _CANONICAL_SYMBOL_RE.fullmatch(stripped) is None:
+ return None
+ return _normalize_symbol(stripped)
def _symbol_from_csv_filename(stem: str) -> str | None:
@@ -441,7 +576,13 @@ def _metric_kind_for_path(path: str) -> str | None:
@dataclass(frozen=True)
class EvidenceRecord:
- """One observed, unavailable, or derived numeric evidence item."""
+ """One observed, unavailable, or derived numeric evidence item.
+
+ ``artifact`` and ``scope`` are set only for backtest output: the file the
+ value was read from and the backtest run directory that produced it, both
+ relative to the ledger's run directory ("" is that directory itself). A
+ declaration's ``ref`` may name either one.
+ """
call_id: str
tool: str
@@ -454,6 +595,8 @@ class EvidenceRecord:
currency: str | None = None
venue: str | None = None
currency_conversion: str | None = None
+ artifact: str | None = None
+ scope: str | None = None
def _is_price_kind(record: EvidenceRecord) -> bool:
@@ -572,6 +715,7 @@ class _EvidenceMixin:
"""Parse metric figures from a successful backtest's run-dir artifacts."""
root = self.run_dir.resolve()
candidates: list[Path] = []
+ own_dir: Path | None = None
raw_dir = arguments.get("run_dir") or payload.get("run_dir")
if raw_dir:
candidate = Path(str(raw_dir))
@@ -581,6 +725,7 @@ class _EvidenceMixin:
resolved = candidate.resolve()
if resolved == root or resolved.is_relative_to(root):
candidates.append(resolved)
+ own_dir = resolved
except OSError:
pass
# The loop archives a detached backtest's artifacts into the active run
@@ -618,8 +763,178 @@ class _EvidenceMixin:
continue
seen_files.add(file_path)
recorded += self._record_metrics_file(file_path, call_id)
+ if recorded:
+ own_dir = own_dir or root
+ scope = _relative_posix(own_dir, root)
+ self._backtest_scopes[call_id] = scope
+ self._scope_latest[scope] = call_id
+ self._record_backtest_outputs(own_dir, call_id, scope)
+ # The active run holds this backtest's copy only when the loop's
+ # archive names it as the source; otherwise it is an earlier run's.
+ if own_dir != root and _archive_source(root) == own_dir.name:
+ self._record_backtest_outputs(root, call_id, scope)
return recorded
+ def _record_backtest_outputs(self, directory: Path, call_id: str, scope: str) -> None:
+ """Record every scalar of a backtest's summary outputs, and hash its tables.
+
+ The run's Sortino, turnover, final equity, Monte Carlo p-values and
+ risk X-ray are as observed as its Sharpe; only kind-named metrics used
+ to count, so a report quoting the rest was refused figure by figure.
+ A file re-recorded under the same path replaces its earlier records:
+ the loop's archive overwrites the active run's copy on every backtest.
+
+ Args:
+ directory: A backtest run directory, or the active run holding its copy.
+ call_id: The backtest call that produced the outputs.
+ scope: That backtest's run directory relative to the ledger's.
+ """
+ root = self.run_dir.resolve()
+ manifest = _run_card_manifest(directory)
+ room = _MAX_GENERIC_EVIDENCE
+ for name in _BACKTEST_SUMMARY_FILES:
+ path = directory / name
+ if not path.is_file() or self._is_model_written(path):
+ continue
+ if name not in _MANIFEST_EXEMPT and manifest.get(name) != _file_sha256(path):
+ continue
+ artifact = _relative_posix(path, root)
+ self._evidence = [record for record in self._evidence if record.artifact != artifact]
+ for field_name, value in _summary_scalars(path):
+ if room <= 0:
+ return
+ if _price_field_for_path(field_name) is not None:
+ continue
+ self._evidence.append(
+ EvidenceRecord(
+ call_id=call_id,
+ tool="backtest",
+ symbol=None,
+ source="backtest",
+ timestamp=None,
+ field=field_name,
+ value=value,
+ status="observed",
+ artifact=artifact,
+ scope=scope,
+ )
+ )
+ room -= 1
+ tables = directory / "artifacts"
+ if tables.is_dir():
+ for path in sorted(tables.glob("*.csv")):
+ if self._is_model_written(path) or f"artifacts/{path.name}" in _BACKTEST_SUMMARY_FILES:
+ continue
+ digest = _file_sha256(path)
+ if digest is not None:
+ self._engine_tables[str(path.resolve())] = (digest, scope)
+
+ def _note_model_write(
+ self, tool_name: str, arguments: Mapping[str, Any], payload: dict[str, Any] | None
+ ) -> None:
+ """Remember a file the model wrote, so it never passes for engine output."""
+ if tool_name not in _WRITE_TOOLS:
+ return
+ raw = (payload or {}).get("path") or arguments.get("path") or arguments.get("file_path")
+ if not raw:
+ return
+ path = Path(str(raw))
+ if not path.is_absolute():
+ path = Path(str(arguments.get("run_dir") or self.run_dir)) / path
+ try:
+ self._model_written.add(str(path.resolve()))
+ except OSError:
+ return
+
+ def _is_model_written(self, path: Path) -> bool:
+ """Whether the model wrote ``path`` itself in this run."""
+ try:
+ return str(path.resolve()) in self._model_written
+ except OSError:
+ return True
+
+ def _ingest_engine_table(self, payload: dict[str, Any], call_id: str) -> None:
+ """Record the rows of a backtest table the model just read back.
+
+ Only a table a completed backtest wrote counts, and only while it is
+ still byte for byte what the engine wrote. Only the rows the model was
+ shown are recorded: a per-bar table is a dense series, and recording
+ all of it would let an invented weight or return match some bar by
+ chance. Price columns are left out, as in a summary file.
+
+ Args:
+ payload: The ``read_file`` result.
+ call_id: The ``read_file`` call.
+ """
+ raw, content = payload.get("path"), payload.get("content")
+ if not isinstance(raw, str) or not isinstance(content, str):
+ return
+ try:
+ path = Path(raw).resolve()
+ except OSError:
+ return
+ known = self._engine_tables.get(str(path))
+ if known is None or _file_sha256(path) != known[0]:
+ return
+ scope = known[1]
+ truncated = "\n... (truncated)"
+ if content.endswith(truncated):
+ content = content[: -len(truncated)]
+ content = content[: content.rfind("\n") + 1]
+ try:
+ rows = list(csv.reader(content.splitlines()))
+ except csv.Error:
+ return
+ if len(rows) < 2:
+ return
+ header = [cell.strip() for cell in rows[0]]
+ folded = [cell.casefold() for cell in header]
+ date_index = next(
+ (index for index, name in enumerate(folded) if name in _CSV_DATE_COLUMNS), None
+ )
+ symbol_index = next(
+ (index for index, name in enumerate(folded) if name in _TABLE_SYMBOL_COLUMNS), None
+ )
+ artifact = _relative_posix(path, self.run_dir.resolve())
+ room = _MAX_GENERIC_EVIDENCE
+ for row in rows[1:]:
+ timestamp = (
+ row[date_index].strip()
+ if date_index is not None and date_index < len(row)
+ else None
+ )
+ for index, cell in enumerate(row[: len(header)]):
+ if index in (date_index, symbol_index):
+ continue
+ value = _coerce_csv_number(cell)
+ if value is None:
+ continue
+ # A weight column is named by its instrument ("000001.SZ"), and
+ # that name is the field a ref narrows on. The record carries no
+ # symbol: a portfolio table is not evidence about the listing,
+ # and a symbol here would start attributing the answer's
+ # portfolio figures to one instrument.
+ column_symbol = _cell_symbol(header[index])
+ if column_symbol is None and _price_field_for_path(folded[index]) is not None:
+ continue
+ if room <= 0:
+ return
+ self._evidence.append(
+ EvidenceRecord(
+ call_id=call_id,
+ tool="read_file",
+ symbol=None,
+ source="backtest",
+ timestamp=timestamp,
+ field=column_symbol or folded[index],
+ value=value,
+ status="observed",
+ artifact=artifact,
+ scope=scope,
+ )
+ )
+ room -= 1
+
def _record_metrics_file(self, path: Path, call_id: str) -> int:
"""Record metric figures from one metrics.csv/metrics.json artifact."""
try:
diff --git a/agent/src/agent/grounding/ledger.py b/agent/src/agent/grounding/ledger.py
index a947d1c25..691e91c02 100644
--- a/agent/src/agent/grounding/ledger.py
+++ b/agent/src/agent/grounding/ledger.py
@@ -100,6 +100,14 @@ class GroundingLedger(
self._symbol_resolution_attempts = 0
self._price_evidence_attempts = 0
self._ingested_csvs: set[str] = set()
+ # Per-bar tables completed backtests wrote: resolved path -> (sha256,
+ # backtest scope), so a later read of one can be recognised as engine
+ # output. Files the model wrote itself never qualify.
+ self._engine_tables: dict[str, tuple[str, str]] = {}
+ self._model_written: set[str] = set()
+ # Backtest call -> its run directory, and each directory's latest call.
+ self._backtest_scopes: dict[str, str] = {}
+ self._scope_latest: dict[str, str] = {}
self._identity_required = bool(_ACTIONABLE_MARKET_RE.search(user_message))
self._buffer_output = self._identity_required
# Every instrument this run is entitled to write about: the ones the
@@ -262,12 +270,15 @@ class GroundingLedger(
return
self._track_session_symbols(arguments, result)
+ self._note_model_write(tool_name, arguments, payload)
if tool_name in _ANALYSIS_TOOLS:
self._ingest_analysis_result(tool_name, arguments, payload, call_id)
if tool_name == _RESOLVER_TOOL:
self._ingest_resolution(arguments, payload, call_id)
elif tool_name == "get_market_data":
self._ingest_market_data(arguments, payload, call_id)
+ elif tool_name == "read_file" and payload is not None:
+ self._ingest_engine_table(payload, call_id)
elif payload is not None:
self._ingest_generic_numeric(tool_name, arguments, payload, call_id)
self.persist()
diff --git a/agent/src/agent/grounding/policies.py b/agent/src/agent/grounding/policies.py
index 2eae3130c..356d57a4a 100644
--- a/agent/src/agent/grounding/policies.py
+++ b/agent/src/agent/grounding/policies.py
@@ -10,7 +10,9 @@ from __future__ import annotations
import ast
import json
import math
+import os
from dataclasses import dataclass, field
+from pathlib import Path
from typing import Any, Callable, Iterable, Mapping, Sequence
from src.agent.grounding.identity import (
@@ -522,6 +524,11 @@ class _PolicyMixin:
if declaration is None:
found = self._check_observed(figure, None, symbol, records)
if block.present and found:
+ # Declared, but as a fraction where the answer writes a
+ # percent (or the reverse): still undeclared, since the
+ # two are different assertions — but say so, or the model
+ # reads "not declared" as a lie and repeats the draft.
+ other_unit = _declared_in_other_unit(block, figure)
issues.append(
self._figure_issue(
"figure_undeclared",
@@ -532,6 +539,7 @@ class _PolicyMixin:
"is not declared in the figures block and is not an observed "
"value; declare it as observed / derived / proposed / cited / "
"count, or remove it",
+ **({"declared_as": other_unit.value_text} if other_unit else {}),
)
)
continue
@@ -663,6 +671,10 @@ class _PolicyMixin:
then its line; the whole-answer fallback is left to the caller.
"""
if figure.symbol:
+ # "000001.SZ 平安银行": the cell names its instrument beside a name.
+ written = _scan_symbols(figure.symbol)
+ if len(written) == 1:
+ return next(iter(written))
normalized = _normalize_symbol(figure.symbol)
if normalized:
return normalized
@@ -679,6 +691,8 @@ class _PolicyMixin:
ref: str,
symbol: str | None,
figure: Figure | None,
+ *,
+ pool_ambiguous: bool = False,
) -> tuple[list[EvidenceRecord], list[float]] | None:
"""The evidence named by an exact field, call+field, one call, or one tool.
@@ -686,44 +700,56 @@ class _PolicyMixin:
one call. When it repeats across calls, ``call_id::field`` is the
unambiguous tightest scope. Otherwise a ``ref`` naming a call id or a
tool name keeps the existing call/tool scope, and the only one that can
- ground a non-price figure (revenue, IC, volume). Records of another
- symbol are dropped when the figure's symbol is known; a currency-marked
- figure keeps only money-denominated records, a percent only the others,
- less metadata counts.
+ ground a non-price figure (revenue, IC, volume). A backtest's output is
+ also named by its run directory or file (:meth:`_artifact_scope`).
+ Records of another symbol are dropped when the figure's symbol is
+ known; a currency-marked figure keeps only money-denominated records, a
+ percent only the others, less metadata counts.
Args:
ref: The declaration's ``ref``.
symbol: The figure's resolved symbol, or None.
figure: The figure whose shape narrows the kind, or None for the
operands of a derivation.
+ pool_ambiguous: Pool a field ref's sources even when they disagree.
+ Only a derivation's anchors ask for it: an operand from either
+ run is still an observation.
Returns:
``(records, metric values)``, or None when ``ref`` names no field,
- call, or tool.
+ call, tool, or backtest output.
"""
key = (ref or "").strip()
if not key:
return None
- # A composite ref names one exact field from one exact call. This is
- # the unambiguous form when the same analysis field appears in more
- # than one tool call during a run.
+ # A composite ref names one exact field from one exact call, or from
+ # one backtest's output (``rp::sharpe``). This is the unambiguous form
+ # when the same analysis field appears in more than one tool call.
if "::" in key:
call_id, field = (part.strip() for part in key.split("::", 1))
if not call_id or not field:
return [], []
records, entries = self._field_sources(field, symbol)
- records = [record for record in records if record.call_id == call_id]
+ by_call = [record for record in records if record.call_id == call_id]
metrics = [float(entry["value"]) for entry in entries if entry.get("call_id") == call_id]
+ # Not a call: ``rp::sharpe`` names a field of one backtest's output,
+ # ``rp::target_positions.csv`` or ``backtest::rp`` the output itself.
+ records = by_call if by_call or metrics else self._artifact_scope(key) or []
else:
records, entries = self._field_sources(key, symbol)
+ artifact_scope = None if records or entries else self._artifact_scope(key)
if records or entries:
- if self._ambiguous_field_sources(key, symbol):
+ if not pool_ambiguous and self._ambiguous_field_sources(key, symbol):
# Calls disagree on this field, so the ref cannot say
# which value it quotes (VaR at 95% vs 99%). Fail closed
# rather than pool them; the issue names the calls.
return [], []
metrics = [float(entry["value"]) for entry in entries]
+ elif artifact_scope is not None:
+ # A backtest's run directory or one of its files: everything
+ # that backtest observed there, and nothing another run did.
+ records, metrics = artifact_scope, []
else:
records = [
record
@@ -815,23 +841,131 @@ class _PolicyMixin:
def _ambiguous_field_sources(self, field: str, symbol: str | None) -> list[str]:
"""The ``call::path`` sources a field-only ref cannot choose between.
- Ambiguous means more than one (call, path) source AND more than one
- value among them: two runs of one call returning the same number leave
- nothing to choose. Each source is written as the ref that names it.
+ Ambiguous means more than one source AND more than one value among
+ them: two runs of one call returning the same number leave nothing to
+ choose. A source is a call and its full path, except that one backtest
+ is one source (:meth:`_ref_source`). Each is written as the ref that
+ names it.
Returns:
- Sorted ``call::path`` refs, or an empty list when the ref is exact.
+ Sorted refs, or an empty list when the ref is exact.
"""
if not field or "::" in field:
return []
records, entries = self._field_sources(field, symbol)
- sources = {(record.call_id, record.field, float(record.value)) for record in records}
+ sources = {
+ (*self._ref_source(record.call_id, record.field, record.scope), float(record.value))
+ for record in records
+ }
sources |= {
- (str(entry.get("call_id")), entry.get("field"), float(entry["value"])) for entry in entries
+ (
+ *self._ref_source(str(entry.get("call_id")), str(entry.get("field")), None),
+ float(entry["value"]),
+ )
+ for entry in entries
}
- if len({(call, path) for call, path, _ in sources}) < 2 or len({value for *_, value in sources}) < 2:
+ if len({identity for identity, *_ in sources}) < 2 or len({value for *_, value in sources}) < 2:
return []
- return sorted({f"{call}::{path}" for call, path, _ in sources})
+ return sorted({label for _, label, _ in sources})
+
+ def _ref_source(
+ self, call_id: str, field: str, scope: str | None
+ ) -> tuple[tuple[str, str], str]:
+ """Which source one evidence value belongs to, and the ref that names it.
+
+ One backtest is one observation: its run card, metrics file and risk
+ X-ray each hold a ``max_drawdown``, and a ref matching several of them
+ still names that one run. Two backtests are two sources, and so is one
+ directory run twice (the earlier call keeps its own identity). A
+ backtest's value is named through its run directory (``rp::sharpe``),
+ which the model chose; anything else by call id and full path.
+
+ Args:
+ call_id: The call that returned the value.
+ field: Its full evidence path.
+ scope: The backtest run directory of a backtest output record.
+
+ Returns:
+ ``(identity, ref label)``.
+ """
+ if scope is None:
+ owner = self._backtest_scopes.get(call_id)
+ if owner is not None and self._scope_latest.get(owner) == call_id:
+ scope = owner
+ if scope is None:
+ return (call_id, field), f"{call_id}::{field}"
+ return ("scope", scope), f"{scope}::{field}" if scope else f"{call_id}::{field}"
+
+ def _artifact_scope(self, key: str) -> list[EvidenceRecord] | None:
+ """The backtest output a ref names by run directory or file, or None.
+
+ Backtest output is named the way the model named the run: its run
+ directory, which the model chose (``rp``, ``risk_parity``), as a path
+ (``rp/artifacts/metrics.csv``, ``rp/metrics.csv``) or beside a file
+ and a field (``risk_parity target_positions.csv``,
+ ``backtest::rp``). The run directory is what makes the ref exact —
+ two backtests hold two different Sharpes — so a ref naming several
+ runs pools them, and a ref naming only a file pools that file across
+ runs, like a ref naming the tool. A file or field the ref also names
+ narrows the set when the run holds it and is ignored otherwise.
+
+ Args:
+ key: The declaration's ref, or one half of ``scope::field``.
+
+ Returns:
+ The observed backtest records the ref names, or None when it names
+ no backtest run directory and no file one wrote.
+ """
+ owned = [
+ record
+ for record in self._evidence
+ if record.artifact is not None
+ and record.status == "observed"
+ and record.value is not None
+ ]
+ if not owned:
+ return None
+ root = self.run_dir.resolve()
+ chosen: dict[int, EvidenceRecord] = {}
+ by_tool: dict[int, EvidenceRecord] = {}
+ files: set[str] = set()
+ fields: list[str] = []
+ for token in _REF_TOKEN_RE.split(key):
+ token = token.strip("`'\"")
+ if not token:
+ continue
+ hits = _records_at(token, owned, root)
+ if hits:
+ chosen.update((id(record), record) for record in hits)
+ elif token != key.strip() and any(record.tool == token for record in owned):
+ # "backtest 两个运行": the tool name beside words. Beside a run
+ # directory ("backtest::rp") it only says what the run is; a
+ # ref that is the tool name alone keeps the tool scope in
+ # ``_referenced``.
+ by_tool.update((id(record), record) for record in owned if record.tool == token)
+ elif "/" not in token and any(
+ _file_stem(token) == _file_stem(record.artifact) for record in owned
+ ):
+ # A bare file name ("target_positions.csv"). A path that holds
+ # none of this session's records names nothing: the active
+ # run's copy may be another backtest's by now.
+ files.add(_file_stem(token))
+ else:
+ fields.append(token)
+ chosen = chosen or by_tool
+ if not chosen and not files:
+ return None
+ selected = list(chosen.values()) or owned
+ by_file = [record for record in selected if _file_stem(record.artifact) in files]
+ selected = by_file or selected
+ for token in fields:
+ by_field = [
+ record
+ for record in selected
+ if record.field == token or record.field.endswith("." + token)
+ ]
+ selected = by_field or selected
+ return selected or None
def _tail_risk_sources(
self,
@@ -855,14 +989,14 @@ class _PolicyMixin:
sources.append((identity, float(entry["value"])))
return sources
- @staticmethod
def _tail_risk_field_refs(
+ self,
records: Sequence[EvidenceRecord],
entries: Iterable[Mapping[str, Any]] = (),
) -> list[str]:
- """Exact call_id::field refs available for tail-risk evidence."""
+ """Exact ``call_id::field`` (or ``run_dir::field``) refs for tail-risk evidence."""
refs = {
- f"{record.call_id}::{record.field}"
+ self._ref_source(record.call_id, record.field, record.scope)[1]
for record in records
if record.call_id
and record.field
@@ -871,7 +1005,7 @@ class _PolicyMixin:
and tail_risk_identity(record.field)
}
refs |= {
- f"{entry.get('call_id')}::{entry.get('field')}"
+ self._ref_source(str(entry.get("call_id")), str(entry.get("field")), None)[1]
for entry in entries
if entry.get("call_id")
and entry.get("field")
@@ -892,12 +1026,12 @@ class _PolicyMixin:
return []
records, entries = self._field_sources(field, symbol)
refs = {
- f"{record.call_id}::{record.field}"
+ self._ref_source(record.call_id, record.field, record.scope)[1]
for record in records
if record.tool == scope and record.call_id and record.field
}
refs |= {
- f"{entry.get('call_id')}::{entry.get('field')}"
+ self._ref_source(str(entry.get("call_id")), str(entry.get("field")), None)[1]
for entry in entries
if entry.get("tool") == scope and entry.get("call_id") and entry.get("field")
}
@@ -1393,32 +1527,43 @@ class _PolicyMixin:
if evaluated is None:
return "formula_not_evaluable"
result, operands, tree = evaluated
- if not symbol and len({record.symbol for record in records if record.symbol}) > 1:
- # Two instruments' bars and no resolved symbol: any arithmetic would
- # look anchored, with nothing to anchor it to.
- return "no_symbol"
- # A money-marked result is derived from money: an RSI or a volume is not
- # a price to take a discount of.
- anchors = self._price_pool(symbol, records) + self._row_pool(symbol, money_only=money)
- if not money:
- anchors += self._metric_pool(symbol)
- scoped = self._referenced(declaration.ref, symbol, None)
+ # Two instruments' bars and no resolved symbol: arithmetic anchored on
+ # the session's pools, or on one instrument's prints the ref names,
+ # would look anchored with nothing to anchor it to. Only evidence that
+ # belongs to no instrument, named by the ref, can anchor it then — a
+ # difference between two backtests' Sharpes is a portfolio figure.
+ unattributed = not symbol and len({record.symbol for record in records if record.symbol}) > 1
+ anchors: list[float] = []
+ if not unattributed:
+ # A money-marked result is derived from money: an RSI or a volume
+ # is not a price to take a discount of.
+ anchors = self._price_pool(symbol, records) + self._row_pool(symbol, money_only=money)
+ if not money:
+ anchors += self._metric_pool(symbol)
+ # Arithmetic across two runs needs both runs' values: a field ref that
+ # names several sources anchors all of them instead of none.
+ scoped = self._referenced(declaration.ref, symbol, None, pool_ambiguous=True)
if scoped is not None:
anchors.extend(
float(record.value)
for record in scoped[0]
- if not money or _is_price_kind(record)
+ if (not money or _is_price_kind(record)) and not (unattributed and record.symbol)
)
- if not money:
+ if not money and not unattributed:
anchors.extend(scoped[1])
if not anchors:
- return "no_evidence"
+ return "no_symbol" if unattributed else "no_evidence"
+
+ # A formula's constants are parsed unsigned ("−0.2099 − (−0.2158)"
+ # holds 0.2099 and 0.2158), so an operand is matched by magnitude, as
+ # a figure is: without it no drawdown or loss could ever anchor.
+ magnitudes = [abs(anchor) for anchor in anchors]
def observed(operand: float) -> bool:
- return _close_any(operand, anchors)
+ return _close_any(abs(operand), magnitudes)
if not any(observed(operand) for operand in operands):
- return "formula_not_anchored"
+ return "no_symbol" if unattributed else "formula_not_anchored"
if _unanchored_term(tree, observed):
return "additive_operand_not_observed"
return result, operands
@@ -1819,6 +1964,82 @@ class _PolicyMixin:
#: The role an undeclared figure is validated under (spec §4, undeclared mode).
+def _declared_in_other_unit(block: FiguresBlock, figure: Figure) -> Declaration | None:
+ """A declaration holding ``figure`` as a fraction where it is a percent, or the reverse.
+
+ It does not cover the figure (``block.match`` keeps the two apart); it
+ only lets the correction say what the model did declare.
+ """
+ half_unit = _written_half_unit(figure.digits or figure.text)
+ for declaration in block.declarations:
+ if declaration.percent == figure.percent:
+ continue
+ in_figure_units = declaration.value * (100.0 if figure.percent else 0.01)
+ if abs(in_figure_units - figure.value) <= half_unit * (1 + 1e-9):
+ return declaration
+ return None
+
+
+#: What separates the parts of a free-form ref: ``backtest::rp``,
+#: ``risk_parity target_positions.csv``, ``backtest(metrics.csv)``.
+_REF_TOKEN_RE = re.compile(r"::?|[\s,,;;()()\[\]]+")
+
+
+def _file_stem(path: str) -> str:
+ """A file's name up to its first dot, casefolded: ``target_positions`` for any spelling."""
+ return str(path).rsplit("/", 1)[-1].split(".", 1)[0].casefold()
+
+
+def _records_at(
+ token: str, owned: Sequence[EvidenceRecord], root: Path
+) -> list[EvidenceRecord]:
+ """The backtest records a path-like ref token names.
+
+ A directory or file relative to the run (or absolute) names what lies at
+ or under it. A file named without its ``artifacts/`` segment
+ (``rp/metrics.csv``) names that file anywhere under its directory, and
+ ``rp/ew`` names both run directories.
+
+ Args:
+ token: One part of a ref.
+ owned: Every observed backtest record.
+ root: The ledger's resolved run directory.
+
+ Returns:
+ The records at that path, or an empty list.
+ """
+ path = Path(token)
+ if not path.is_absolute():
+ path = root / path
+ try:
+ relative = Path(os.path.normpath(path)).relative_to(root).as_posix()
+ except ValueError:
+ return []
+ if relative in ("", "."):
+ return []
+
+ def under(artifact: str, prefix: str) -> bool:
+ return artifact == prefix or artifact.startswith(prefix + "/")
+
+ hits = [record for record in owned if under(record.artifact or "", relative)]
+ if hits or "/" not in relative:
+ return hits
+ parent, _, name = relative.rpartition("/")
+ hits = [
+ record
+ for record in owned
+ if under(record.artifact or "", parent)
+ and _file_stem(record.artifact or "") == _file_stem(name)
+ ]
+ if hits:
+ return hits
+ groups = [
+ [record for record in owned if under(record.artifact or "", part)]
+ for part in relative.split("/")
+ ]
+ return [record for group in groups for record in group] if all(groups) else []
+
+
_NO_DECLARATION = Declaration(
index=0, value_text="", value=0.0, percent=False, role="observed", note="", ref=""
)
diff --git a/agent/src/agent/grounding/release.py b/agent/src/agent/grounding/release.py
index a24957473..1c988b687 100644
--- a/agent/src/agent/grounding/release.py
+++ b/agent/src/agent/grounding/release.py
@@ -122,6 +122,12 @@ def _correction_line(issue: dict[str, Any]) -> str:
sources=", ".join(str(source) for source in issue.get("ambiguous_sources") or []),
result=result if result else "a different value",
)
+ declared_as = issue.get("declared_as")
+ if declared_as:
+ evidence += (
+ f"; the figures block declares {declared_as}, which is not how the answer "
+ f"writes it — declare it exactly as written ({issue.get('value')})"
+ )
candidates = issue.get("field_ref_candidates") or []
if candidates:
evidence += "; valid field refs: " + ", ".join(str(item) for item in candidates)
@@ -212,6 +218,16 @@ class _ReleaseMixin:
"report it as not retrieved instead.",
]
)
+ if self._backtest_scopes:
+ runs = ", ".join(f"`{scope}`" for scope in sorted(set(self._backtest_scopes.values())) if scope)
+ lines.append(
+ "A value a backtest wrote is observed with that backtest's run directory "
+ "as ref"
+ + (f" ({runs})" if runs else "")
+ + ", or the file you read under it; a difference between two backtests "
+ "is derived, with both run directories as ref. A figure the answer writes "
+ "as a percent is declared as a percent."
+ )
recovery = self.recovery_action(validation)
if recovery == _RESOLVER_TOOL:
lines.extend(
@@ -334,7 +350,7 @@ class _ReleaseMixin:
for validation in self._validations
for code in (issue.get("code") for issue in validation.get("issues", []))
}
- if issue_codes & _REDACTABLE_CODES:
+ if issue_codes & _REDACTABLE_CODES and not self._analysis_completed:
if is_zh:
return (
"我的回答被安全门槛拒绝:草稿引用了本会话未通过工具获取的价格数字,无法核验。"
@@ -346,6 +362,23 @@ class _ReleaseMixin:
"be verified. Re-run the task and let the agent fetch the market data first, "
"or ask it to answer without the unverified prices."
)
+ if issue_codes & _REDACTABLE_CODES:
+ # A completed analysis whose report failed the check: the draft's
+ # figures, not prices, are what failed, and the run's output is
+ # not lost with it.
+ if is_zh:
+ return (
+ "我的回答没有通过数字核验:草稿里有数字无法与本会话工具返回的结果对上,"
+ "按规则没有发布。分析本身已经完成,结果文件保存在本次运行的 artifacts "
+ "目录中。可以重试,或让我只列出工具直接返回的数字。"
+ )
+ return (
+ "My answer did not pass the figure check: some of its figures could not be "
+ "matched to what this session's tools returned, so it was not released. "
+ "The analysis itself completed; its result files are in this run's "
+ "artifacts directory. Retry, or ask me to list only the figures the tools "
+ "returned."
+ )
if is_zh:
return (
"当前无法安全确认标的身份或价格证据,因此没有生成交易结论。"
@@ -491,8 +524,11 @@ class _ReleaseMixin:
with a note stating how many figures were removed, or None.
"""
# A market answer with no observed price has nothing to stand on once its
- # figures are cut; a general answer (no instrument asked about) does.
- if self._identity_required and not self._price_records():
+ # figures are cut; a general answer (no instrument asked about) does, and
+ # so does a completed analysis: naming 600519.SH in a backtest request
+ # makes it a market answer, but the backtest's own output is what the
+ # surviving figures were checked against.
+ if self._identity_required and not self._price_records() and not self._analysis_completed:
return None
text = _strip_release_markers(content)
# Stripping shifts offsets and the cuts anchor on issue spans, so the
diff --git a/agent/src/agent/loop.py b/agent/src/agent/loop.py
index 7da096491..3fe856b13 100644
--- a/agent/src/agent/loop.py
+++ b/agent/src/agent/loop.py
@@ -30,6 +30,7 @@ from typing import Any, Callable, Dict, List, Mapping, Optional
from src.agent.context import ContextBuilder
from src.agent.grounding import GroundingLedger
+from src.agent.grounding.evidence import ARCHIVE_MANIFEST as _ARCHIVE_MANIFEST
from src.agent.grounding.release import MAX_GROUNDING_REVISIONS
from src.agent.memory import WorkspaceMemory
from src.agent.progress import HeartbeatTimer, ProgressEvent, _set_emitter
@@ -1005,9 +1006,9 @@ def _normalize_tool_run_dir(args: dict[str, Any], memory_run_dir: str | None) ->
return normalized
-#: Names the backtest an archived run currently describes, and the files that
-#: archive placed there, so the next one can replace exactly its own output.
-_ARCHIVE_MANIFEST = ".archived_backtest.json"
+# ``_ARCHIVE_MANIFEST`` names the backtest an archived run currently describes,
+# and the files that archive placed there, so the next one can replace exactly
+# its own output. The grounding ledger reads it to tell whose copy it is.
def _previously_archived(target: Path) -> set[str]:
diff --git a/agent/tests/test_grounding_backtest_outputs.py b/agent/tests/test_grounding_backtest_outputs.py
new file mode 100644
index 000000000..1f2d0ce72
--- /dev/null
+++ b/agent/tests/test_grounding_backtest_outputs.py
@@ -0,0 +1,529 @@
+"""A backtest's own output grounds the report written about it.
+
+Regression history (2026-09-29): a reporter's risk-parity vs equal-weight
+backtest came back as the canned refusal. Replaying the 15 DeepSeek runs of that
+prompt through the gate showed why: 4 of 5 rejected figures were values the
+backtest itself wrote — Sortino, Calmar, turnover, final value, Monte Carlo
+p-values, the optimiser's weights — because only kind-named metrics (Sharpe,
+return, drawdown …) ever became evidence, and a ref naming the run
+(``rp/artifacts/metrics.csv``, ``risk_parity target_positions.csv``) resolved to
+nothing. Declared figures: 196 ``value_mismatch`` and 167
+``not_in_referenced_call`` became 1 and 7. And naming 600519.SH in the request
+made it a market answer, so a report with no fetched price could never be
+released with its failing figures cut: 12 of 15 runs ended in the refusal.
+
+What stays closed is pinned beside each change: an undeclared figure is still
+checked only against prices and metric kinds (widening it to the whole metrics
+row let 16% of invented decimals through, measured on the same runs), a file
+the model wrote is never evidence, a table counts only for the rows the model
+was shown, and a value from one backtest does not ground a figure declared from
+another.
+
+Every value here is synthetic; no market data lives in the repository.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+from pathlib import Path
+
+import pytest
+
+from src.agent.grounding import GroundingLedger
+from src.agent.grounding.evidence import ARCHIVE_MANIFEST
+
+RP = {
+ "final_value": 1129817.32,
+ "total_return": 0.129817,
+ "annual_return": 0.135530,
+ "max_drawdown": -0.211044,
+ "sharpe": 0.693563,
+ "calmar": 0.6422,
+ "sortino": 1.1329,
+ "trade_count": 10,
+ "total_turnover": 1.366493,
+}
+EW = {
+ "final_value": 1132804.94,
+ "total_return": 0.132805,
+ "annual_return": 0.138657,
+ "max_drawdown": -0.214919,
+ "sharpe": 0.704341,
+ "calmar": 0.6452,
+ "sortino": 1.2115,
+ "trade_count": 8,
+ "total_turnover": 1.045117,
+}
+RP_WEIGHTS = [
+ ("2024-01-02", 0.389729, 0.300167, 0.310103),
+ ("2024-01-03", 0.389840, 0.297964, 0.312195),
+ ("2024-01-04", 0.390124, 0.297953, 0.311923),
+ ("2024-01-05", 0.371544, 0.286110, 0.342346),
+]
+
+
+def _sha(path: Path) -> str:
+ return hashlib.sha256(path.read_bytes()).hexdigest()
+
+
+def _write_backtest(
+ run_dir: Path,
+ metrics: dict[str, float],
+ *,
+ weights: list[tuple[str, float, float, float]] | None = None,
+ xray_drawdown: float = -0.218469,
+ manifest_skips: tuple[str, ...] = (),
+) -> None:
+ """Write the files a completed backtest leaves behind, run card last."""
+ artifacts = run_dir / "artifacts"
+ artifacts.mkdir(parents=True, exist_ok=True)
+ names = list(metrics)
+ (artifacts / "metrics.csv").write_text(
+ ",".join(names) + "\n" + ",".join(str(metrics[name]) for name in names) + "\n",
+ encoding="utf-8",
+ )
+ (artifacts / "risk_xray.json").write_text(
+ json.dumps(
+ {
+ "concentration": {"hhi": 0.340460, "effective_n": 2.937202},
+ "drawdown": {"max_drawdown": xray_drawdown},
+ "tail_risk": {"var_95": 0.022701, "expected_shortfall_95": 0.031423},
+ "correlation": {"avg_pairwise_abs": 0.696128},
+ }
+ ),
+ encoding="utf-8",
+ )
+ (artifacts / "validation.json").write_text(
+ json.dumps(
+ {
+ "monte_carlo": {
+ "p_value_sharpe": 0.304,
+ "p_value_max_dd": 0.439,
+ # A series: 1,000 samples would match any Sharpe-sized figure.
+ "sharpe_samples": [round(0.001 * index, 3) for index in range(1000)],
+ }
+ }
+ ),
+ encoding="utf-8",
+ )
+ rows = weights or RP_WEIGHTS
+ (artifacts / "target_positions.csv").write_text(
+ "timestamp,000001.SZ,000858.SZ,600519.SH\n"
+ + "".join(f"{day},{a},{b},{c}\n" for day, a, b, c in rows),
+ encoding="utf-8",
+ )
+ (artifacts / "trades.csv").write_text(
+ "timestamp,code,side,price,qty,pnl\n"
+ "2024-03-01,000001.SZ,sell,9.81,1000,11488.33\n"
+ "2024-06-03,600519.SH,sell,1602.5,100,-17998.19\n",
+ encoding="utf-8",
+ )
+ listed = [
+ f"artifacts/{name}"
+ for name in ("metrics.csv", "risk_xray.json", "validation.json", "target_positions.csv", "trades.csv")
+ if f"artifacts/{name}" not in manifest_skips
+ ]
+ card = {
+ "backtest": {"initial_cash": 1000000, "start_date": "2024-01-01"},
+ "metrics": dict(metrics),
+ "artifacts": [{"path": name, "sha256": _sha(run_dir / name)} for name in listed],
+ "citations": [{"row": 1, "metric": "sharpe"}],
+ }
+ (run_dir / "run_card.json").write_text(json.dumps(card), encoding="utf-8")
+
+
+def _backtest(ledger: GroundingLedger, run_dir: Path, call_id: str) -> None:
+ ledger.ingest_tool_result(
+ tool_name="backtest",
+ arguments={"run_dir": str(run_dir)},
+ result=json.dumps({"status": "ok", "exit_code": 0, "run_dir": str(run_dir)}),
+ call_id=call_id,
+ success=True,
+ )
+
+
+def _read(ledger: GroundingLedger, path: Path, call_id: str, *, limit: int | None = None) -> None:
+ text = path.read_text(encoding="utf-8")
+ if limit:
+ text = "".join(text.splitlines(keepends=True)[:limit])
+ ledger.ingest_tool_result(
+ tool_name="read_file",
+ arguments={"path": str(path)},
+ result=json.dumps({"status": "ok", "path": str(path), "content": text}),
+ call_id=call_id,
+ success=True,
+ )
+
+
+def _declared(prose: str, *lines: str) -> str:
+ return prose + "\n\n```figures\n" + "\n".join(lines) + "\n```"
+
+
+def _figure_reasons(result) -> list[str]:
+ return [issue["reason"] for issue in result.issues if issue.get("value") is not None]
+
+
+def two_runs(tmp_path: Path) -> GroundingLedger:
+ """A risk-parity run in ``rp/`` and an equal-weight run in ``ew/``."""
+ _write_backtest(tmp_path / "rp", RP)
+ _write_backtest(
+ tmp_path / "ew",
+ EW,
+ weights=[("2024-01-02", 0.333333, 0.333333, 0.333334)],
+ xray_drawdown=-0.227175,
+ )
+ ledger = GroundingLedger(
+ run_dir=tmp_path,
+ user_message="用000001.SZ、600519.SH、000858.SZ构建风险平价组合,回测2024全年,与等权基准对比",
+ )
+ _backtest(ledger, tmp_path / "rp", "bt-rp")
+ _backtest(ledger, tmp_path / "ew", "bt-ew")
+ return ledger
+
+
+def test_every_scalar_a_backtest_reports_is_observed_under_its_run_directory(
+ two_runs: GroundingLedger,
+) -> None:
+ """Sortino, Calmar, turnover, p-values, X-ray and config echo, not just Sharpe."""
+ result = two_runs.validate_final_answer(
+ _declared(
+ "风险平价 Sortino 1.133,Calmar 0.6422,累计换手 1.366,有效持仓 2.937,"
+ "HHI 0.3405,蒙特卡洛 Sharpe p 值 0.304,平均两两相关 0.696。",
+ "1.133 | observed | Sortino | rp",
+ "0.6422 | observed | Calmar | rp",
+ "1.366 | observed | 累计换手 | rp",
+ "2.937 | observed | 有效持仓 | rp",
+ "0.3405 | observed | HHI | rp",
+ "0.304 | observed | p 值 | rp",
+ "0.696 | observed | 相关 | rp",
+ )
+ )
+
+ assert result.valid, result.issues
+
+
+ "ref",
+ [
+ "rp",
+ "rp/artifacts/metrics.csv",
+ "rp/metrics.csv",
+ "rp metrics.csv",
+ "backtest::rp",
+ "rp::sortino",
+ "backtest rp",
+ ],
+)
+def test_every_spelling_of_a_run_names_that_run_and_no_other(
+ two_runs: GroundingLedger, ref: str
+) -> None:
+ """The run directory is what makes a ref exact: EW's Sortino is not RP's."""
+ own = two_runs.validate_final_answer(
+ _declared("风险平价 Sortino 1.133。", f"1.133 | observed | Sortino | {ref}")
+ )
+ other = two_runs.validate_final_answer(
+ _declared("风险平价 Sortino 1.212。", f"1.212 | observed | Sortino | {ref}")
+ )
+
+ assert own.valid, own.issues
+ assert not other.valid
+ assert {issue["reason"] for issue in other.issues} == {"not_in_referenced_call"}
+
+
+def test_an_invented_value_declared_from_a_real_run_is_rejected(
+ two_runs: GroundingLedger,
+) -> None:
+ """Naming the run is not a pass: the value must be one it wrote."""
+ result = two_runs.validate_final_answer(
+ _declared("风险平价 Sortino 1.190。", "1.190 | observed | Sortino | rp")
+ )
+ # 0.512 is one of the 1,000 Monte Carlo samples: a series is not a result.
+ sampled = two_runs.validate_final_answer(
+ _declared("模拟 Sharpe 0.512。", "0.512 | observed | Sharpe | rp")
+ )
+
+ assert not result.valid
+ assert [issue["value"] for issue in result.issues] == ["1.190"]
+ assert not sampled.valid
+
+
+def test_a_bare_field_two_runs_hold_is_ambiguous_and_the_correction_names_the_runs(
+ two_runs: GroundingLedger,
+) -> None:
+ result = two_runs.validate_final_answer(
+ _declared("风险平价 Sortino 1.133。", "1.133 | observed | Sortino | sortino")
+ )
+
+ assert not result.valid
+ (issue,) = result.issues
+ assert issue["reason"] == "ambiguous_field_ref"
+ assert "rp::sortino" in issue["field_ref_candidates"]
+ assert "ew::sortino" in issue["field_ref_candidates"]
+ assert "rp::sortino" in two_runs.correction_prompt(result)
+
+
+def test_one_run_holding_two_drawdowns_is_still_one_source(tmp_path: Path) -> None:
+ """Engine and risk X-ray both report a max_drawdown; a ref cannot be ambiguous
+ between two numbers the same run wrote."""
+ _write_backtest(tmp_path / "rp", RP)
+ ledger = GroundingLedger(run_dir=tmp_path, user_message="回测风险平价")
+ _backtest(ledger, tmp_path / "rp", "bt-rp")
+
+ for written in ("−21.10%", "−21.85%"):
+ result = ledger.validate_final_answer(
+ _declared(f"最大回撤 {written}。", f"{written} | observed | 最大回撤 | max_drawdown")
+ )
+ assert result.valid, (written, result.issues)
+
+
+def test_a_table_grounds_only_the_rows_the_model_was_shown(two_runs: GroundingLedger) -> None:
+ table = two_runs.run_dir / "rp" / "artifacts" / "target_positions.csv"
+ shown = _declared(
+ "2024-01-02 风险平价权重:000001.SZ 38.97%。",
+ "38.97% | observed | 初始权重 | rp/artifacts/target_positions.csv",
+ )
+ unseen = _declared(
+ "2024-01-05 风险平价权重:600519.SH 34.23%。",
+ "34.23% | observed | 权重 | rp/artifacts/target_positions.csv",
+ )
+
+ assert not two_runs.validate_final_answer(shown).valid # not read yet
+ _read(two_runs, table, "read-1", limit=3)
+
+ assert two_runs.validate_final_answer(shown).valid
+ assert not two_runs.validate_final_answer(unseen).valid
+
+
+def test_a_file_the_model_wrote_is_never_engine_output(tmp_path: Path) -> None:
+ """Neither a table written before the backtest nor one edited after it."""
+ ledger = GroundingLedger(run_dir=tmp_path, user_message="回测风险平价")
+ own = tmp_path / "rp" / "artifacts" / "my_weights.csv"
+ own.parent.mkdir(parents=True)
+ own.write_text("timestamp,000001.SZ\n2024-01-02,0.4567\n", encoding="utf-8")
+ ledger.ingest_tool_result(
+ tool_name="write_file",
+ arguments={"path": str(own)},
+ result=json.dumps({"status": "ok", "path": str(own)}),
+ call_id="w1",
+ success=True,
+ )
+ own.unlink()
+ _write_backtest(tmp_path / "rp", RP)
+ own.write_text("timestamp,000001.SZ\n2024-01-02,0.4567\n", encoding="utf-8")
+ _backtest(ledger, tmp_path / "rp", "bt-rp")
+ _read(ledger, own, "read-own")
+
+ forged = _declared("权重 45.67%。", "45.67% | observed | 权重 | rp/artifacts/my_weights.csv")
+ assert not ledger.validate_final_answer(forged).valid
+
+ table = tmp_path / "rp" / "artifacts" / "target_positions.csv"
+ table.write_text("timestamp,000001.SZ\n2024-01-02,0.4567\n", encoding="utf-8")
+ _read(ledger, table, "read-edited")
+ edited = _declared("权重 45.67%。", "45.67% | observed | 权重 | rp/artifacts/target_positions.csv")
+ assert not ledger.validate_final_answer(edited).valid
+
+
+def test_a_summary_file_the_run_card_does_not_vouch_for_is_ignored(tmp_path: Path) -> None:
+ _write_backtest(tmp_path / "rp", RP, manifest_skips=("artifacts/risk_xray.json",))
+ ledger = GroundingLedger(run_dir=tmp_path, user_message="回测风险平价")
+ _backtest(ledger, tmp_path / "rp", "bt-rp")
+
+ unvouched = ledger.validate_final_answer(
+ _declared("平均两两相关 0.696。", "0.696 | observed | 相关 | rp")
+ )
+ vouched = ledger.validate_final_answer(
+ _declared("Sortino 1.133。", "1.133 | observed | Sortino | rp")
+ )
+
+ assert not unvouched.valid
+ assert vouched.valid, vouched.issues
+
+
+def test_the_active_runs_copy_belongs_to_the_backtest_the_archive_names(tmp_path: Path) -> None:
+ """The loop copies each finished backtest into the active run; a ref to that
+ copy names the backtest it came from, and only once it has come."""
+ ledger = GroundingLedger(run_dir=tmp_path, user_message="回测")
+ _write_backtest(tmp_path / "rp", RP)
+ _write_backtest(tmp_path / "ew", EW)
+
+ def archive(source: str) -> None:
+ for name in ("metrics.csv", "risk_xray.json", "validation.json"):
+ target = tmp_path / "artifacts" / name
+ target.parent.mkdir(exist_ok=True)
+ target.write_bytes((tmp_path / source / "artifacts" / name).read_bytes())
+ (tmp_path / "run_card.json").write_bytes((tmp_path / source / "run_card.json").read_bytes())
+ (tmp_path / ARCHIVE_MANIFEST).write_text(json.dumps({"source_run": source}), encoding="utf-8")
+
+ def copy_says(value: str) -> bool:
+ return ledger.validate_final_answer(
+ _declared(f"Sortino {value}。", f"{value} | observed | Sortino | artifacts/metrics.csv")
+ ).valid
+
+ archive("ew") # an earlier archive: not rp's output, and not credited to rp
+ _backtest(ledger, tmp_path / "rp", "bt-rp")
+ assert not copy_says("1.133")
+ assert not copy_says("1.212")
+
+ archive("rp")
+ _backtest(ledger, tmp_path / "rp", "bt-rp-2")
+ assert copy_says("1.133")
+
+ archive("ew")
+ _backtest(ledger, tmp_path / "ew", "bt-ew")
+ assert copy_says("1.212")
+ assert not copy_says("1.133")
+
+
+def test_an_undeclared_figure_is_checked_exactly_as_before(two_runs: GroundingLedger) -> None:
+ """The undeclared path still accepts only prices and metric kinds.
+
+ Widening it to the whole metrics row accepted 16% of invented decimals on
+ the replayed runs, against 4.6% today; the model declares the rest.
+ """
+ sharpe = two_runs.validate_final_answer("风险平价 Sharpe 0.694。")
+ sortino = two_runs.validate_final_answer("风险平价 Sortino 1.133。")
+
+ assert sharpe.valid, sharpe.issues
+ assert not sortino.valid
+ assert [issue["value"] for issue in sortino.issues] == ["1.133"]
+
+
+def test_a_percent_declared_as_a_fraction_is_named_in_the_correction(
+ two_runs: GroundingLedger,
+) -> None:
+ """37% and 0.37 stay two assertions (spec §2), but the model is told which it wrote."""
+ result = two_runs.validate_final_answer(
+ _declared("总收益差 +0.30pp。", "0.0030 | derived | 0.132805 − 0.129817 | rp, ew")
+ )
+
+ assert not result.valid
+ (issue,) = result.issues
+ assert issue["code"] == "figure_undeclared"
+ assert issue["declared_as"] == "0.0030"
+ assert "declares 0.0030" in two_runs.correction_prompt(result)
+
+
+def test_a_drawdown_difference_anchors_on_magnitudes(two_runs: GroundingLedger) -> None:
+ """Formula constants parse unsigned; a negative observation must still anchor them."""
+ result = two_runs.validate_final_answer(
+ _declared(
+ "风险平价回撤浅 0.39pp。",
+ "0.39pp | derived | (−0.211044) − (−0.214919) | rp, ew",
+ )
+ )
+
+ assert result.valid, result.issues
+
+
+def test_arithmetic_across_two_runs_anchors_on_a_field_ref_naming_both(
+ two_runs: GroundingLedger,
+) -> None:
+ """For a derivation, a field both runs hold is two observations, not an ambiguity."""
+ result = two_runs.validate_final_answer(
+ _declared("等权 Sortino 高 0.079。", "0.079 | derived | 1.2115 − 1.1329 | sortino")
+ )
+ unanchored = two_runs.validate_final_answer(
+ _declared("等权 Sortino 高 0.079。", "0.079 | derived | 1.2115 − 1.1329 | 索提诺差")
+ )
+
+ assert result.valid, result.issues
+ assert not unanchored.valid
+
+
+def test_a_named_derivation_survives_prices_of_several_symbols(two_runs: GroundingLedger) -> None:
+ """Several instruments' bars bar the session pools, not the evidence a ref names."""
+ two_runs.ingest_tool_result(
+ tool_name="get_market_data",
+ arguments={"codes": ["000001.SZ", "600519.SH"]},
+ result=json.dumps(
+ {
+ "000001.SZ": [{"trade_date": "2024-12-31", "close": 11.2}],
+ "600519.SH": [{"trade_date": "2024-12-31", "close": 1524.0}],
+ }
+ ),
+ call_id="md",
+ success=True,
+ )
+ named = two_runs.validate_final_answer(
+ _declared("Sortino 差 0.079。", "0.079 | derived | 1.2115 − 1.1329 | rp, ew")
+ )
+ unnamed = two_runs.validate_final_answer(
+ _declared("Sharpe 差 0.011。", "0.011 | derived | 0.704341 − 0.693563 | 夏普差")
+ )
+
+ # The prose names no symbol, source or currency, which the bars now ask
+ # for; only the figures' own verdicts are under test.
+ assert _figure_reasons(named) == [], named.issues
+ assert _figure_reasons(unnamed) == ["no_symbol"]
+
+
+def test_a_backtest_report_is_released_cut_instead_of_refused(two_runs: GroundingLedger) -> None:
+ """A request naming symbols is a market answer; a completed backtest still
+ leaves the checked figures standing once the failing one is cut."""
+ draft = "风险平价 Sharpe 0.694,Sortino 1.190。"
+ validation = two_runs.validate_final_answer(draft)
+
+ released = two_runs.redacted_release(draft, validation)
+
+ assert two_runs._identity_required
+ assert released is not None
+ assert "0.694" in released and "1.190" not in released and "(略※)" in released
+
+
+def test_a_market_answer_with_no_price_and_no_analysis_is_still_refused(tmp_path: Path) -> None:
+ ledger = GroundingLedger(run_dir=tmp_path, user_message="600519.SH 现价多少,给出买入价")
+ draft = "600519.SH 买入价 1500.5 元。"
+
+ assert ledger.redacted_release(draft, ledger.validate_final_answer(draft)) is None
+ assert "价格数字" in ledger.safe_fallback()
+
+
+def test_the_fallback_after_an_analysis_names_the_figures_not_prices(
+ two_runs: GroundingLedger,
+) -> None:
+ two_runs.validate_final_answer("风险平价 Sortino 1.190。")
+
+ message = two_runs.safe_fallback()
+
+ assert "价格" not in message
+ assert "artifacts" in message
+
+
+def test_a_code_and_name_cell_attributes_its_figure_to_the_code(tmp_path: Path) -> None:
+ """"000001.SZ 平安银行" in the symbol column is 000001.SZ, not a mismatch."""
+ ledger = GroundingLedger(run_dir=tmp_path, user_message="000001.SZ 和 600519.SH 最近收盘")
+ ledger.ingest_tool_result(
+ tool_name="get_market_data",
+ arguments={"codes": ["000001.SZ", "600519.SH"]},
+ result=json.dumps(
+ {
+ "000001.SZ": [{"trade_date": "2024-12-31", "close": 11.2}],
+ "600519.SH": [{"trade_date": "2024-12-31", "close": 1524.0}],
+ }
+ ),
+ call_id="md",
+ success=True,
+ )
+ result = ledger.validate_final_answer(
+ "| 标的 | 收盘 |\n|---|---:|\n| 000001.SZ 平安银行 | 11.20 |\n\n"
+ "数据来源 auto,人民币计价。\n\n```figures\n11.20 | observed | 000001.SZ 收盘 | md\n```"
+ )
+
+ assert not any(issue.get("reason") == "symbol_mismatch" for issue in result.issues), result.issues
+
+
+def test_a_trade_price_read_back_is_not_a_market_print(two_runs: GroundingLedger) -> None:
+ """An execution price is left out of the table evidence, or reading trades.csv
+ would turn a backtest report into a price answer owing a source and currency."""
+ _read(two_runs, two_runs.run_dir / "rp" / "artifacts" / "trades.csv", "read-trades")
+
+ result = two_runs.validate_final_answer(
+ _declared("风险平价 Sortino 1.133,平安银行一笔盈利 11488.33。",
+ "1.133 | observed | Sortino | rp",
+ "11488.33 | observed | 盈亏 | rp/artifacts/trades.csv")
+ )
+
+ assert result.valid, result.issues
+ assert not two_runs._price_records()