"Vibe-Trading: Your Personal Trading Agent"
Python ai-agentalgorithmic-tradingbacktestingfintechllmmcpmulti-agentpythonquantitative-financetrading
"""Agent tool: research-report data audit (extract → verify → verdict).
A ``BaseTool`` wrapper around the report-audit routines that guard published
research against hallucinated numbers. Auto-discovered and registered via
``BaseTool.__subclasses__()``.
Two-phase quality gate:
1. ``extract`` — parse a markdown report, collect its numeric data points
(tables, ``label: value`` lines, ``亿 / 万亿 / x / 倍 / %`` units), and
draw a random sample (default 15%, clamped to [3, 30]).
2. ``verdict`` — compare each sampled point's reported value against one or two
authoritative fetched values at a 1% tolerance; a single failed point fails
the whole report, mixed results warn.
The tool takes text/results only — it does not fetch data itself. Pair it with
market-data / financial-statement tools: read the report, extract here, fetch
there, then verdict here. Read-only: returns JSON, writes nothing.
"""
from __future__ import annotations
import json
import math
import re
from random import Random
from typing import Any
from src.agent.tools import BaseTool
# ---------------------------------------------------------------------------
# Markdown data-point extraction (handles Chinese financial reports)
# ---------------------------------------------------------------------------
_KV_LABEL_RE = re.compile(r"(?P<label>[一-龥A-Za-z][^|\n::*]{1,30})[::]\s*")
_UNIT_PATTERN = r"亿[元美港]?元?|万亿|倍|%|[BMTxX](?![A-Za-z])"
_NUMUNIT_RE = re.compile(
r"[~约]?\$?\s*(?:"
r"(?P<open>[((])\s*\$?\s*(?P<account_num>[+\-−]?[\d,,.]+)\s*"
rf"(?P<inside_unit>{_UNIT_PATTERN})?\s*(?P<close>[))])\s*"
rf"(?P<outside_unit>{_UNIT_PATTERN})?"
r"|(?P<currency_sign>[+\-−])?\s*\$?\s*(?P<num>[+\-−]?[\d,,.]+)\s*"
rf"(?P<unit>{_UNIT_PATTERN})?)"
)
_TABLE_SEP_RE = re.compile(r"^\|[\-\s|:]+\|$")
_SKIP_LABELS = {
"来源", "sources", "source", "说明", "注意", "备注", "数据来源",
"n/a", "—", "-", "/", "合计", "total", "单位", "趋势",
}
_QUARTER_RE = re.compile(r"(20\d{2}|Q[1-4]|\d{4}\s*Q[1-4])")
def _clean_num(s: str) -> float | None:
"""Normalise a numeric string with commas (ASCII or wide) to float."""
s = s.replace(",", "").replace(",", "").replace("−", "-").strip()
try:
return float(s)
except ValueError:
return None
def _report_value(text: str, *, prose: bool = False) -> tuple[float, str, int] | None:
"""Read a financial value without rescuing malformed accounting notation.
Args:
text: Table cell or text immediately following a prose label.
prose: Require a unit or currency for a parenthesized prose value.
Returns:
Finite signed value, its unit and consumed end offset, or None.
"""
match = _NUMUNIT_RE.match(text) if prose else _NUMUNIT_RE.search(text)
if match is None:
return None
# A failed accounting match must not restart at the unsigned inner digits.
prefix = text[: match.start()]
if prefix.count("(") > prefix.count(")") or prefix.count("(") > prefix.count(")"):
return None
suffix = text[match.end() :].lstrip()
if suffix.startswith((")", ")")):
return None
if match.group("open"):
if (match.group("open"), match.group("close")) not in {
("(", ")"),
("(", ")"),
}:
return None
inside, outside = match.group("inside_unit"), match.group("outside_unit")
if inside and outside:
return None
unit = inside or outside or ""
if prose and not unit and "$" not in match.group():
return None
value = _clean_num(match.group("account_num"))
if value is not None:
# Parens mean negative regardless of an inner sign: "(-25)" is
# the same debt as "(25)", never a credit.
value = -abs(value)
else:
value = _clean_num(match.group("num"))
sign = match.group("currency_sign")
if sign:
if match.group("num").startswith(("+", "-", "−")):
return None
if value is not None and sign in {"-", "−"}:
value = -value
unit = match.group("unit") or ""
if value is None or not math.isfinite(value):
return None
return value, unit, match.end()
def _is_valid_label(label: str) -> bool:
"""True if a label looks like a meaningful financial field name."""
label = label.strip()
if len(label) < 2:
return False
if re.fullmatch(r"[\d\s年季度Q]+", label):
return False
if re.match(r"^[+\-*#|~$>_`]", label):
return False
if "**" in label or "`" in label or "__" in label:
return False
if re.fullmatch(r"[+-]?\d+(\.\d+)?%", label):
return False
if label.lower() in _SKIP_LABELS:
return False
return True
def _split_md_table_row(line: str) -> list[str]:
"""Split a markdown table row, keeping empty leading/trailing/middle cells.
Filtering empties would shift later values onto earlier headers
(``| Profit | | 25M |`` must stay under Q2, not Q1).
"""
parts = line.strip().split("|")
if parts and parts[0] == "":
parts = parts[1:]
if parts and parts[-1] == "":
parts = parts[:-1]
return [c.strip().strip("*_~").strip() for c in parts]
def _parse_md_tables(lines: list[str]) -> list[tuple[str, str, float, str, int, str]]:
"""Parse markdown tables into (row_label, col_header, value, unit, lineno, raw)."""
results: list[tuple[str, str, float, str, int, str]] = []
i = 0
while i < len(lines):
line = lines[i].strip()
if "|" in line and not _TABLE_SEP_RE.match(line):
headers_raw = [
h.strip().strip("*_").strip() for h in _split_md_table_row(line)
]
if i + 1 < len(lines) and _TABLE_SEP_RE.match(lines[i + 1].strip()):
i += 2 # skip the separator row
while i < len(lines):
dline = lines[i].strip()
if not dline or not dline.startswith("|"):
break
cells = _split_md_table_row(dline)
if len(cells) < 2:
i += 1
continue
row_label = cells[0]
for col_idx, cell in enumerate(cells[1:], start=1):
if not cell:
continue
col_header = (
headers_raw[col_idx]
if col_idx < len(headers_raw)
else f"col{col_idx}"
)
parsed = _report_value(cell)
if parsed is not None:
val, unit, _ = parsed
if val != 0 and abs(val) < 1e15:
results.append(
(row_label, col_header, val, unit, i + 1, dline)
)
i += 1
continue
i += 1
return results
def _without_fenced_code(lines: list[str]) -> list[str]:
"""Blank fenced examples without changing the report's line numbers."""
result = []
fence = ""
for line in lines:
match = re.match(r"^ {0,3}(`{3,}|~{3,})(.*)$", line)
if fence:
result.append("")
if (
match
and match[1][0] == fence[0]
and len(match[1]) >= len(fence)
and not match[2].strip()
):
fence = ""
continue
if match and (match[1][0] != "`" or "`" not in match[2]):
fence = match[1]
result.append("")
else:
result.append(line)
return result
def extract_data_points(md_text: str) -> list[dict[str, Any]]:
"""Extract recognizable financial data points from a markdown report.
Covers multi-column tables and ``label: value unit`` lines. De-duplicates
by (label, value, unit).
Args:
md_text: Full markdown text of the report.
Returns:
List of point dicts: ``{id, label, reported_value, unit, raw_text,
line_number}``.
"""
points: list[dict[str, Any]] = []
seen: set[str] = set()
def _add(label: str, val: float | None, unit: str, lineno: int, raw: str) -> None:
label = re.sub(r"[*_`]+", "", label).strip()
if not _is_valid_label(label):
return
if val is None or not math.isfinite(val) or val == 0 or abs(val) > 1e15:
return
if _QUARTER_RE.fullmatch(label.strip()):
return
key = f"{label}|{round(val, 4)}|{unit}"
if key in seen:
return
seen.add(key)
points.append(
{
"id": len(points) + 1,
"label": label,
"reported_value": val,
"unit": unit,
"raw_text": raw[:120],
"line_number": lineno,
}
)
lines = _without_fenced_code(md_text.split("\n"))
# 1. Multi-column tables.
_YOY_HEADERS = {"YOY", "YOY增速", "增速", "同比", "变化", "趋势", "说明", "备注"}
for row_label, col_header, val, unit, lineno, raw in _parse_md_tables(lines):
if not _is_valid_label(row_label):
continue
if col_header.upper() in _YOY_HEADERS:
continue
label = (
f"{row_label} · {col_header}"
if (col_header and col_header != row_label)
else row_label
)
_add(label, val, unit, lineno, raw)
# 2. ``label: value unit`` lines.
for lineno, line in enumerate(lines, start=1):
stripped = line.strip()
if stripped.startswith("> ") or re.match(r"^#{1,6}\s", stripped):
continue
if "|" in stripped:
continue # handled as a table above
cursor = 0
while (m := _KV_LABEL_RE.search(stripped, cursor)) is not None:
cursor = m.end()
parsed = _report_value(stripped[m.end() :], prose=True)
if parsed is not None:
val, unit, consumed = parsed
cursor += consumed
_add(m.group("label"), val, unit, lineno, stripped)
return points
def sample_points(
points: list[dict[str, Any]], ratio: float = 0.15, seed: int | None = None,
) -> list[dict[str, Any]]:
"""Draw a random sample, clamped to [3, 30], ordered by line number.
Args:
points: Data points returned by :func:`extract_data_points`.
ratio: Sampling fraction.
seed: Optional seed for reproducibility.
Returns:
Sampled points sorted by ``line_number``.
"""
n = max(3, min(30, math.ceil(len(points) * ratio)))
n = min(n, len(points))
if n == 0:
return []
rng = Random(seed)
sampled = rng.sample(points, n)
return sorted(sampled, key=lambda p: p["line_number"])
# ---------------------------------------------------------------------------
# Verdict
# ---------------------------------------------------------------------------
_TOLERANCE = 0.01 # 1% absolute relative tolerance
def _pct_diff(reported: float, fetched: float) -> float:
"""Absolute relative percent difference (inf when reported is 0 and fetched isn't)."""
if reported == 0:
return 0.0 if fetched == 0 else float("inf")
return abs(reported - fetched) / abs(reported)
def _pct_diff_for_json(diff: float) -> float | None:
"""Convert a relative diff to a JSON-safe percent (null when non-finite)."""
if not math.isfinite(diff):
return None
return round(diff * 100, 2)
def _finite_number(value: Any) -> float | None:
"""Coerce a verification value to a finite float, or None when unusable.
Args:
value: Raw ``reported_value``/``fetched_value`` from a result item.
Returns:
The value as a finite float, or ``None`` when it is missing, not
numeric, or non-finite (NaN/Infinity) — i.e. cannot be verified.
A genuine reported ``0`` is a valid number and returns ``0.0``.
"""
try:
number = float(value)
except (TypeError, ValueError, OverflowError):
return None
return number if math.isfinite(number) else None
def render_verdict(results: list[dict[str, Any]], report_name: str = "") -> dict[str, Any]:
"""Render a PASS/FAIL verdict from per-point verification results.
Each result with a ``fetched_value`` is judged against the reported value
at :data:`_TOLERANCE`. A second source (``fetched_value2``) may be supplied;
both sources must pass for a point to pass, both fail to fail, otherwise the
point warns (treated as a caliber mismatch, not a failure).
A point whose ``reported_value`` is missing, ``None``, or non-finite is
unverifiable, so it fails with an explicit ``reason`` instead of being
treated as a reported zero. A genuine reported ``0`` stays valid. A
supplied but non-finite ``fetched_value`` fails the point the same way
(unverifiable evidence), while an absent one is skipped as "not verified".
Args:
results: List of verification objects — ``{id, label, reported_value,
unit, fetched_value, fetched_source, (optional) fetched_value2,
fetched_source2, ...}``.
report_name: Optional report name for display.
Returns:
Verdict dict: ``verdict`` is ``PASS`` (zero failures, at least one
verified point) or ``FAIL``. A call in which nothing was verified
fails closed — a report certified with zero evidence is the failure
mode this gate exists to prevent.
"""
fail_items: list[dict[str, Any]] = []
warn_items: list[dict[str, Any]] = []
pass_count = 0
total = 0
for item in results:
raw_fetched = item.get("fetched_value")
if raw_fetched is None:
continue # not verified — skip, do not count (pinned design)
# Coerce through _finite_number so a non-numeric fetched value (a
# string, NaN, …) is treated as unusable evidence, not a crash.
fetched = _finite_number(raw_fetched)
if fetched is None:
# Supplied but unusable is NOT "not verified" (which is silently
# skipped): fail the point. Otherwise a junk fetch could be used
# to drop a hard-to-verify number from the audit entirely.
total += 1
label = item.get("label", "?")
fail_items.append({
"id": item.get("id"), "label": label,
"reported": _finite_number(item.get("reported_value")),
"unit": item.get("unit", ""),
"fetched": None, "source": item.get("fetched_source", "?"),
"fetched2": None, "source2": item.get("fetched_source2", ""),
"diff1_pct": None, "diff2_pct": None,
"reason": (
f"fetched value is not a finite number ({raw_fetched!r}) "
"— cannot verify"
),
"raw_text": item.get("raw_text", ""),
"line_number": item.get("line_number", 0),
})
continue
total += 1
label = item.get("label", "?")
unit = item.get("unit", "")
source = item.get("fetched_source", "?")
# A missing/non-numeric reported value is unverifiable evidence, NOT a
# reported zero: fabricating 0.0 here would let `None` vs fetched 0 pass
# the tolerance check and falsely certify the report.
raw_reported = item.get("reported_value")
reported = _finite_number(raw_reported)
if reported is None:
fail_items.append({
"id": item.get("id"), "label": label,
"reported": None, "unit": unit,
"fetched": fetched, "source": source,
"fetched2": _finite_number(item.get("fetched_value2")),
"source2": item.get("fetched_source2", ""),
"diff1_pct": None, "diff2_pct": None,
"reason": (
"reported value missing — cannot verify"
if raw_reported is None
else f"reported value is not a finite number ({raw_reported!r}) — cannot verify"
),
"raw_text": item.get("raw_text", ""),
"line_number": item.get("line_number", 0),
})
continue
diff1 = _pct_diff(reported, fetched)
# A non-finite second source is ignored (single-source semantics)
# instead of crashing the verdict; a garbage source verifies nothing.
fetched2 = _finite_number(item.get("fetched_value2"))
source2 = item.get("fetched_source2", "")
pass1 = diff1 <= _TOLERANCE
if fetched2 is None:
# Single source: pass or fail outright.
if pass1:
pass_count += 1
else:
fail_items.append({
"id": item.get("id"), "label": label,
"reported": reported, "unit": unit,
"fetched": fetched, "source": source,
"fetched2": None, "source2": source2,
"diff1_pct": _pct_diff_for_json(diff1), "diff2_pct": None,
"raw_text": item.get("raw_text", ""),
"line_number": item.get("line_number", 0),
})
else:
# Two sources: PASS only if both agree within tolerance; FAIL if
# both miss; otherwise WARN (a caliber / GAAP mismatch, not a fail).
f2 = fetched2 # already coerced finite above
diff2 = _pct_diff(reported, f2)
pass2 = diff2 <= _TOLERANCE
if pass1 and pass2:
pass_count += 1
elif not pass1 and not pass2:
fail_items.append({
"id": item.get("id"), "label": label,
"reported": reported, "unit": unit,
"fetched": fetched, "source": source,
"fetched2": f2, "source2": source2,
"diff1_pct": _pct_diff_for_json(diff1),
"diff2_pct": _pct_diff_for_json(diff2),
"raw_text": item.get("raw_text", ""),
"line_number": item.get("line_number", 0),
})
else:
warn_items.append({
"id": item.get("id"), "label": label,
"reported": reported, "unit": unit,
"diff1_pct": _pct_diff_for_json(diff1),
"diff2_pct": _pct_diff_for_json(diff2),
})
if total == 0:
# Fail closed: zero verified points means the audit gathered no
# evidence at all. Certifying that as PASS would let an unaudited
# report — or an agent that fetched nothing — through the gate.
fail_items.append({
"id": None, "label": "audit", "reported": None, "unit": "",
"fetched": None, "source": "", "fetched2": None, "source2": "",
"diff1_pct": None, "diff2_pct": None,
"reason": "no data points were verified — every result lacked a fetched_value",
"raw_text": "", "line_number": 0,
})
fail_count = len(fail_items)
verdict = "PASS" if fail_count == 0 else "FAIL"
return {
"verdict": verdict,
"report_name": report_name,
"total": total,
"pass_count": pass_count,
"warn_count": len(warn_items),
"fail_count": fail_count,
"fail_items": fail_items,
"warn_items": warn_items,
}
class ReportAuditTool(BaseTool):
"""Research-report data audit gate (extract sample → verdict)."""
name = "report_audit"
description = (
"Audit a research report's numeric data points for accuracy before "
"publishing. Two sub-commands via `command`: 'extract' parses a "
"markdown report and returns a random sample (~15%, clamped 3-30) of "
"its financial data points to verify; 'verdict' compares each sampled "
"point's reported value against one or two authoritative fetched values "
"at 1% tolerance and returns a PASS/FAIL gate (one failure fails the "
"report). Workflow: read the report, extract here, verify each sample "
"point against market-data/financial-statement tools, then verdict here."
)
parameters = {
"type": "object",
"properties": {
"command": {
"type": "string",
"enum": ["extract", "verdict"],
"description": "Which audit phase to run.",
},
"report_text": {
"type": "string",
"description": "extract: full markdown text of the report to audit.",
},
"ratio": {
"type": "number", "default": 0.15,
"description": "extract: fraction of data points to sample.",
},
"seed": {
"type": "integer",
"description": "extract: random seed for reproducible sampling.",
},
"results": {
"type": "array", "items": {"type": "object"},
"description": "verdict: list of {id, label, reported_value, unit, "
"fetched_value, fetched_source, (optional) "
"fetched_value2, fetched_source2}.",
},
"report_name": {
"type": "string",
"description": "verdict: report name for display.",
},
},
"required": ["command"],
}
is_readonly = True
repeatable = True # loop.py dedups non-repeatable tools by name; extract
# and verdict are commonly called back-to-back.
def execute(self, **kwargs: Any) -> str:
"""Dispatch to ``extract`` or ``verdict`` and return a JSON envelope.
Args:
**kwargs: ``command`` plus the inputs for that phase.
Returns:
JSON string — ``status="ok"`` with the result on success,
``status="error"`` with a message otherwise.
"""
command = str(kwargs.get("command") or "").strip()
try:
if command == "extract":
report_text = kwargs.get("report_text")
if not isinstance(report_text, str) or not report_text.strip():
return _err("report_text (non-empty markdown) is required for extract")
raw_ratio = kwargs.get("ratio")
try:
ratio = float(raw_ratio) if raw_ratio is not None and raw_ratio != "" else 0.15
except (TypeError, ValueError, OverflowError):
return _err(f"invalid ratio: {raw_ratio!r}")
raw_seed = kwargs.get("seed")
try:
seed = int(raw_seed) if raw_seed is not None and raw_seed != "" else None
except (TypeError, ValueError, OverflowError):
return _err(f"invalid seed: {raw_seed!r}")
points = extract_data_points(report_text)
sampled = sample_points(points, ratio=ratio, seed=seed)
result: dict[str, Any] = {
"total_extracted": len(points),
"sample_size": len(sampled),
"ratio": ratio,
"seed": seed,
"sample": sampled,
"hint": (
"For each point in `sample`, fetch its value from an "
"authoritative source, then call the 'verdict' command "
"with results=[{id, reported_value, fetched_value, "
"fetched_source, ...}]."
),
}
elif command == "verdict":
results = kwargs.get("results")
if not isinstance(results, list) or not results:
return _err("results (non-empty list) is required for verdict")
result = render_verdict(
results, report_name=str(kwargs.get("report_name") or ""),
)
else:
return _err(f"unknown command: {command}")
except Exception as exc: # noqa: BLE001 - surface a clean tool error
return json.dumps(
{"status": "error", "command": command, "error": str(exc)},
ensure_ascii=False,
allow_nan=False,
)
return json.dumps(
{"status": "ok", "command": command, **result},
ensure_ascii=False,
allow_nan=False,
)
def _err(msg: str) -> str:
return json.dumps(
{"status": "error", "error": msg}, ensure_ascii=False, allow_nan=False,
)