Skip to content

"Vibe-Trading: Your Personal Trading Agent"

24.6 KB
Raw Blame
"""Agent tool: research-report data audit (extract → verify → verdict).

A ``BaseTool`` wrapper around the report-audit routines that guard published
research against hallucinated numbers. Auto-discovered and registered via
``BaseTool.__subclasses__()``.

Two-phase quality gate:

1. ``extract`` — parse a markdown report, collect its numeric data points
   (tables, ``label: value`` lines, ``亿 / 万亿 / x / 倍 / %`` units), and
   draw a random sample (default 15%, clamped to [3, 30]).
2. ``verdict`` — compare each sampled point's reported value against one or two
   authoritative fetched values at a 1% tolerance; a single failed point fails
   the whole report, mixed results warn.

The tool takes text/results only — it does not fetch data itself. Pair it with
market-data / financial-statement tools: read the report, extract here, fetch
there, then verdict here. Read-only: returns JSON, writes nothing.
"""

from __future__ import annotations

import json
import math
import re
from random import Random
from typing import Any

from src.agent.tools import BaseTool

# ---------------------------------------------------------------------------
# Markdown data-point extraction (handles Chinese financial reports)
# ---------------------------------------------------------------------------

_KV_LABEL_RE = re.compile(r"(?P<label>[一-龥A-Za-z][^|\n::*]{1,30})[::]\s*")
_UNIT_PATTERN = r"亿[元美港]?元?|万亿|倍|%|[BMTxX](?![A-Za-z])"
_NUMUNIT_RE = re.compile(
    r"[~约]?\$?\s*(?:"
    r"(?P<open>[((])\s*\$?\s*(?P<account_num>[+\-−]?[\d,,.]+)\s*"
    rf"(?P<inside_unit>{_UNIT_PATTERN})?\s*(?P<close>[))])\s*"
    rf"(?P<outside_unit>{_UNIT_PATTERN})?"
    r"|(?P<currency_sign>[+\-−])?\s*\$?\s*(?P<num>[+\-−]?[\d,,.]+)\s*"
    rf"(?P<unit>{_UNIT_PATTERN})?)"
)
_TABLE_SEP_RE = re.compile(r"^\|[\-\s|:]+\|$")

_SKIP_LABELS = {
    "来源", "sources", "source", "说明", "注意", "备注", "数据来源",
    "n/a", "—", "-", "/", "合计", "total", "单位", "趋势",
}

_QUARTER_RE = re.compile(r"(20\d{2}|Q[1-4]|\d{4}\s*Q[1-4])")


def _clean_num(s: str) -> float | None:
    """Normalise a numeric string with commas (ASCII or wide) to float."""
    s = s.replace(",", "").replace(",", "").replace("−", "-").strip()
    try:
        return float(s)
    except ValueError:
        return None


def _report_value(text: str, *, prose: bool = False) -> tuple[float, str, int] | None:
    """Read a financial value without rescuing malformed accounting notation.

    Args:
        text: Table cell or text immediately following a prose label.
        prose: Require a unit or currency for a parenthesized prose value.

    Returns:
        Finite signed value, its unit and consumed end offset, or None.
    """
    match = _NUMUNIT_RE.match(text) if prose else _NUMUNIT_RE.search(text)
    if match is None:
        return None
    # A failed accounting match must not restart at the unsigned inner digits.
    prefix = text[: match.start()]
    if prefix.count("(") > prefix.count(")") or prefix.count("(") > prefix.count(")"):
        return None
    suffix = text[match.end() :].lstrip()
    if suffix.startswith((")", ")")):
        return None
    if match.group("open"):
        if (match.group("open"), match.group("close")) not in {
            ("(", ")"),
            ("(", ")"),
        }:
            return None
        inside, outside = match.group("inside_unit"), match.group("outside_unit")
        if inside and outside:
            return None
        unit = inside or outside or ""
        if prose and not unit and "$" not in match.group():
            return None
        value = _clean_num(match.group("account_num"))
        if value is not None:
            # Parens mean negative regardless of an inner sign: "(-25)" is
            # the same debt as "(25)", never a credit.
            value = -abs(value)
    else:
        value = _clean_num(match.group("num"))
        sign = match.group("currency_sign")
        if sign:
            if match.group("num").startswith(("+", "-", "−")):
                return None
            if value is not None and sign in {"-", "−"}:
                value = -value
        unit = match.group("unit") or ""
    if value is None or not math.isfinite(value):
        return None
    return value, unit, match.end()


def _is_valid_label(label: str) -> bool:
    """True if a label looks like a meaningful financial field name."""
    label = label.strip()
    if len(label) < 2:
        return False
    if re.fullmatch(r"[\d\s年季度Q]+", label):
        return False
    if re.match(r"^[+\-*#|~$>_`]", label):
        return False
    if "**" in label or "`" in label or "__" in label:
        return False
    if re.fullmatch(r"[+-]?\d+(\.\d+)?%", label):
        return False
    if label.lower() in _SKIP_LABELS:
        return False
    return True


def _split_md_table_row(line: str) -> list[str]:
    """Split a markdown table row, keeping empty leading/trailing/middle cells.

    Filtering empties would shift later values onto earlier headers
    (``| Profit |  | 25M |`` must stay under Q2, not Q1).
    """
    parts = line.strip().split("|")
    if parts and parts[0] == "":
        parts = parts[1:]
    if parts and parts[-1] == "":
        parts = parts[:-1]
    return [c.strip().strip("*_~").strip() for c in parts]


def _parse_md_tables(lines: list[str]) -> list[tuple[str, str, float, str, int, str]]:
    """Parse markdown tables into (row_label, col_header, value, unit, lineno, raw)."""
    results: list[tuple[str, str, float, str, int, str]] = []
    i = 0
    while i < len(lines):
        line = lines[i].strip()
        if "|" in line and not _TABLE_SEP_RE.match(line):
            headers_raw = [
                h.strip().strip("*_").strip() for h in _split_md_table_row(line)
            ]
            if i + 1 < len(lines) and _TABLE_SEP_RE.match(lines[i + 1].strip()):
                i += 2  # skip the separator row
                while i < len(lines):
                    dline = lines[i].strip()
                    if not dline or not dline.startswith("|"):
                        break
                    cells = _split_md_table_row(dline)
                    if len(cells) < 2:
                        i += 1
                        continue
                    row_label = cells[0]
                    for col_idx, cell in enumerate(cells[1:], start=1):
                        if not cell:
                            continue
                        col_header = (
                            headers_raw[col_idx]
                            if col_idx < len(headers_raw)
                            else f"col{col_idx}"
                        )
                        parsed = _report_value(cell)
                        if parsed is not None:
                            val, unit, _ = parsed
                            if val != 0 and abs(val) < 1e15:
                                results.append(
                                    (row_label, col_header, val, unit, i + 1, dline)
                                )
                    i += 1
                continue
        i += 1
    return results


def _without_fenced_code(lines: list[str]) -> list[str]:
    """Blank fenced examples without changing the report's line numbers."""
    result = []
    fence = ""
    for line in lines:
        match = re.match(r"^ {0,3}(`{3,}|~{3,})(.*)$", line)
        if fence:
            result.append("")
            if (
                match
                and match[1][0] == fence[0]
                and len(match[1]) >= len(fence)
                and not match[2].strip()
            ):
                fence = ""
            continue
        if match and (match[1][0] != "`" or "`" not in match[2]):
            fence = match[1]
            result.append("")
        else:
            result.append(line)
    return result


def extract_data_points(md_text: str) -> list[dict[str, Any]]:
    """Extract recognizable financial data points from a markdown report.

    Covers multi-column tables and ``label: value unit`` lines. De-duplicates
    by (label, value, unit).

    Args:
        md_text: Full markdown text of the report.

    Returns:
        List of point dicts: ``{id, label, reported_value, unit, raw_text,
        line_number}``.
    """
    points: list[dict[str, Any]] = []
    seen: set[str] = set()

    def _add(label: str, val: float | None, unit: str, lineno: int, raw: str) -> None:
        label = re.sub(r"[*_`]+", "", label).strip()
        if not _is_valid_label(label):
            return
        if val is None or not math.isfinite(val) or val == 0 or abs(val) > 1e15:
            return
        if _QUARTER_RE.fullmatch(label.strip()):
            return
        key = f"{label}|{round(val, 4)}|{unit}"
        if key in seen:
            return
        seen.add(key)
        points.append(
            {
                "id": len(points) + 1,
                "label": label,
                "reported_value": val,
                "unit": unit,
                "raw_text": raw[:120],
                "line_number": lineno,
            }
        )

    lines = _without_fenced_code(md_text.split("\n"))

    # 1. Multi-column tables.
    _YOY_HEADERS = {"YOY", "YOY增速", "增速", "同比", "变化", "趋势", "说明", "备注"}
    for row_label, col_header, val, unit, lineno, raw in _parse_md_tables(lines):
        if not _is_valid_label(row_label):
            continue
        if col_header.upper() in _YOY_HEADERS:
            continue
        label = (
            f"{row_label} · {col_header}"
            if (col_header and col_header != row_label)
            else row_label
        )
        _add(label, val, unit, lineno, raw)

    # 2. ``label: value unit`` lines.
    for lineno, line in enumerate(lines, start=1):
        stripped = line.strip()
        if stripped.startswith("> ") or re.match(r"^#{1,6}\s", stripped):
            continue
        if "|" in stripped:
            continue  # handled as a table above
        cursor = 0
        while (m := _KV_LABEL_RE.search(stripped, cursor)) is not None:
            cursor = m.end()
            parsed = _report_value(stripped[m.end() :], prose=True)
            if parsed is not None:
                val, unit, consumed = parsed
                cursor += consumed
                _add(m.group("label"), val, unit, lineno, stripped)

    return points


def sample_points(
    points: list[dict[str, Any]], ratio: float = 0.15, seed: int | None = None,
) -> list[dict[str, Any]]:
    """Draw a random sample, clamped to [3, 30], ordered by line number.

    Args:
        points: Data points returned by :func:`extract_data_points`.
        ratio: Sampling fraction.
        seed: Optional seed for reproducibility.

    Returns:
        Sampled points sorted by ``line_number``.
    """
    n = max(3, min(30, math.ceil(len(points) * ratio)))
    n = min(n, len(points))
    if n == 0:
        return []
    rng = Random(seed)
    sampled = rng.sample(points, n)
    return sorted(sampled, key=lambda p: p["line_number"])


# ---------------------------------------------------------------------------
# Verdict
# ---------------------------------------------------------------------------

_TOLERANCE = 0.01  # 1% absolute relative tolerance


def _pct_diff(reported: float, fetched: float) -> float:
    """Absolute relative percent difference (inf when reported is 0 and fetched isn't)."""
    if reported == 0:
        return 0.0 if fetched == 0 else float("inf")
    return abs(reported - fetched) / abs(reported)


def _pct_diff_for_json(diff: float) -> float | None:
    """Convert a relative diff to a JSON-safe percent (null when non-finite)."""
    if not math.isfinite(diff):
        return None
    return round(diff * 100, 2)


def _finite_number(value: Any) -> float | None:
    """Coerce a verification value to a finite float, or None when unusable.

    Args:
        value: Raw ``reported_value``/``fetched_value`` from a result item.

    Returns:
        The value as a finite float, or ``None`` when it is missing, not
        numeric, or non-finite (NaN/Infinity) — i.e. cannot be verified.
        A genuine reported ``0`` is a valid number and returns ``0.0``.
    """
    try:
        number = float(value)
    except (TypeError, ValueError, OverflowError):
        return None
    return number if math.isfinite(number) else None


def render_verdict(results: list[dict[str, Any]], report_name: str = "") -> dict[str, Any]:
    """Render a PASS/FAIL verdict from per-point verification results.

    Each result with a ``fetched_value`` is judged against the reported value
    at :data:`_TOLERANCE`. A second source (``fetched_value2``) may be supplied;
    both sources must pass for a point to pass, both fail to fail, otherwise the
    point warns (treated as a caliber mismatch, not a failure).

    A point whose ``reported_value`` is missing, ``None``, or non-finite is
    unverifiable, so it fails with an explicit ``reason`` instead of being
    treated as a reported zero. A genuine reported ``0`` stays valid. A
    supplied but non-finite ``fetched_value`` fails the point the same way
    (unverifiable evidence), while an absent one is skipped as "not verified".

    Args:
        results: List of verification objects — ``{id, label, reported_value,
            unit, fetched_value, fetched_source, (optional) fetched_value2,
            fetched_source2, ...}``.
        report_name: Optional report name for display.

    Returns:
        Verdict dict: ``verdict`` is ``PASS`` (zero failures, at least one
        verified point) or ``FAIL``. A call in which nothing was verified
        fails closed — a report certified with zero evidence is the failure
        mode this gate exists to prevent.
    """
    fail_items: list[dict[str, Any]] = []
    warn_items: list[dict[str, Any]] = []
    pass_count = 0
    total = 0

    for item in results:
        raw_fetched = item.get("fetched_value")
        if raw_fetched is None:
            continue  # not verified — skip, do not count (pinned design)
        # Coerce through _finite_number so a non-numeric fetched value (a
        # string, NaN, …) is treated as unusable evidence, not a crash.
        fetched = _finite_number(raw_fetched)
        if fetched is None:
            # Supplied but unusable is NOT "not verified" (which is silently
            # skipped): fail the point. Otherwise a junk fetch could be used
            # to drop a hard-to-verify number from the audit entirely.
            total += 1
            label = item.get("label", "?")
            fail_items.append({
                "id": item.get("id"), "label": label,
                "reported": _finite_number(item.get("reported_value")),
                "unit": item.get("unit", ""),
                "fetched": None, "source": item.get("fetched_source", "?"),
                "fetched2": None, "source2": item.get("fetched_source2", ""),
                "diff1_pct": None, "diff2_pct": None,
                "reason": (
                    f"fetched value is not a finite number ({raw_fetched!r}) "
                    "— cannot verify"
                ),
                "raw_text": item.get("raw_text", ""),
                "line_number": item.get("line_number", 0),
            })
            continue
        total += 1
        label = item.get("label", "?")
        unit = item.get("unit", "")
        source = item.get("fetched_source", "?")

        # A missing/non-numeric reported value is unverifiable evidence, NOT a
        # reported zero: fabricating 0.0 here would let `None` vs fetched 0 pass
        # the tolerance check and falsely certify the report.
        raw_reported = item.get("reported_value")
        reported = _finite_number(raw_reported)
        if reported is None:
            fail_items.append({
                "id": item.get("id"), "label": label,
                "reported": None, "unit": unit,
                "fetched": fetched, "source": source,
                "fetched2": _finite_number(item.get("fetched_value2")),
                "source2": item.get("fetched_source2", ""),
                "diff1_pct": None, "diff2_pct": None,
                "reason": (
                    "reported value missing — cannot verify"
                    if raw_reported is None
                    else f"reported value is not a finite number ({raw_reported!r}) — cannot verify"
                ),
                "raw_text": item.get("raw_text", ""),
                "line_number": item.get("line_number", 0),
            })
            continue

        diff1 = _pct_diff(reported, fetched)

        # A non-finite second source is ignored (single-source semantics)
        # instead of crashing the verdict; a garbage source verifies nothing.
        fetched2 = _finite_number(item.get("fetched_value2"))
        source2 = item.get("fetched_source2", "")
        pass1 = diff1 <= _TOLERANCE

        if fetched2 is None:
            # Single source: pass or fail outright.
            if pass1:
                pass_count += 1
            else:
                fail_items.append({
                    "id": item.get("id"), "label": label,
                    "reported": reported, "unit": unit,
                    "fetched": fetched, "source": source,
                    "fetched2": None, "source2": source2,
                    "diff1_pct": _pct_diff_for_json(diff1), "diff2_pct": None,
                    "raw_text": item.get("raw_text", ""),
                    "line_number": item.get("line_number", 0),
                })
        else:
            # Two sources: PASS only if both agree within tolerance; FAIL if
            # both miss; otherwise WARN (a caliber / GAAP mismatch, not a fail).
            f2 = fetched2  # already coerced finite above
            diff2 = _pct_diff(reported, f2)
            pass2 = diff2 <= _TOLERANCE
            if pass1 and pass2:
                pass_count += 1
            elif not pass1 and not pass2:
                fail_items.append({
                    "id": item.get("id"), "label": label,
                    "reported": reported, "unit": unit,
                    "fetched": fetched, "source": source,
                    "fetched2": f2, "source2": source2,
                    "diff1_pct": _pct_diff_for_json(diff1),
                    "diff2_pct": _pct_diff_for_json(diff2),
                    "raw_text": item.get("raw_text", ""),
                    "line_number": item.get("line_number", 0),
                })
            else:
                warn_items.append({
                    "id": item.get("id"), "label": label,
                    "reported": reported, "unit": unit,
                    "diff1_pct": _pct_diff_for_json(diff1),
                    "diff2_pct": _pct_diff_for_json(diff2),
                })

    if total == 0:
        # Fail closed: zero verified points means the audit gathered no
        # evidence at all. Certifying that as PASS would let an unaudited
        # report — or an agent that fetched nothing — through the gate.
        fail_items.append({
            "id": None, "label": "audit", "reported": None, "unit": "",
            "fetched": None, "source": "", "fetched2": None, "source2": "",
            "diff1_pct": None, "diff2_pct": None,
            "reason": "no data points were verified — every result lacked a fetched_value",
            "raw_text": "", "line_number": 0,
        })

    fail_count = len(fail_items)
    verdict = "PASS" if fail_count == 0 else "FAIL"
    return {
        "verdict": verdict,
        "report_name": report_name,
        "total": total,
        "pass_count": pass_count,
        "warn_count": len(warn_items),
        "fail_count": fail_count,
        "fail_items": fail_items,
        "warn_items": warn_items,
    }


class ReportAuditTool(BaseTool):
    """Research-report data audit gate (extract sample → verdict)."""

    name = "report_audit"
    description = (
        "Audit a research report's numeric data points for accuracy before "
        "publishing. Two sub-commands via `command`: 'extract' parses a "
        "markdown report and returns a random sample (~15%, clamped 3-30) of "
        "its financial data points to verify; 'verdict' compares each sampled "
        "point's reported value against one or two authoritative fetched values "
        "at 1% tolerance and returns a PASS/FAIL gate (one failure fails the "
        "report). Workflow: read the report, extract here, verify each sample "
        "point against market-data/financial-statement tools, then verdict here."
    )
    parameters = {
        "type": "object",
        "properties": {
            "command": {
                "type": "string",
                "enum": ["extract", "verdict"],
                "description": "Which audit phase to run.",
            },
            "report_text": {
                "type": "string",
                "description": "extract: full markdown text of the report to audit.",
            },
            "ratio": {
                "type": "number", "default": 0.15,
                "description": "extract: fraction of data points to sample.",
            },
            "seed": {
                "type": "integer",
                "description": "extract: random seed for reproducible sampling.",
            },
            "results": {
                "type": "array", "items": {"type": "object"},
                "description": "verdict: list of {id, label, reported_value, unit, "
                               "fetched_value, fetched_source, (optional) "
                               "fetched_value2, fetched_source2}.",
            },
            "report_name": {
                "type": "string",
                "description": "verdict: report name for display.",
            },
        },
        "required": ["command"],
    }
    is_readonly = True
    repeatable = True  # loop.py dedups non-repeatable tools by name; extract
                       # and verdict are commonly called back-to-back.

    def execute(self, **kwargs: Any) -> str:
        """Dispatch to ``extract`` or ``verdict`` and return a JSON envelope.

        Args:
            **kwargs: ``command`` plus the inputs for that phase.

        Returns:
            JSON string — ``status="ok"`` with the result on success,
            ``status="error"`` with a message otherwise.
        """
        command = str(kwargs.get("command") or "").strip()
        try:
            if command == "extract":
                report_text = kwargs.get("report_text")
                if not isinstance(report_text, str) or not report_text.strip():
                    return _err("report_text (non-empty markdown) is required for extract")
                raw_ratio = kwargs.get("ratio")
                try:
                    ratio = float(raw_ratio) if raw_ratio is not None and raw_ratio != "" else 0.15
                except (TypeError, ValueError, OverflowError):
                    return _err(f"invalid ratio: {raw_ratio!r}")
                raw_seed = kwargs.get("seed")
                try:
                    seed = int(raw_seed) if raw_seed is not None and raw_seed != "" else None
                except (TypeError, ValueError, OverflowError):
                    return _err(f"invalid seed: {raw_seed!r}")
                points = extract_data_points(report_text)
                sampled = sample_points(points, ratio=ratio, seed=seed)
                result: dict[str, Any] = {
                    "total_extracted": len(points),
                    "sample_size": len(sampled),
                    "ratio": ratio,
                    "seed": seed,
                    "sample": sampled,
                    "hint": (
                        "For each point in `sample`, fetch its value from an "
                        "authoritative source, then call the 'verdict' command "
                        "with results=[{id, reported_value, fetched_value, "
                        "fetched_source, ...}]."
                    ),
                }
            elif command == "verdict":
                results = kwargs.get("results")
                if not isinstance(results, list) or not results:
                    return _err("results (non-empty list) is required for verdict")
                result = render_verdict(
                    results, report_name=str(kwargs.get("report_name") or ""),
                )
            else:
                return _err(f"unknown command: {command}")
        except Exception as exc:  # noqa: BLE001 - surface a clean tool error
            return json.dumps(
                {"status": "error", "command": command, "error": str(exc)},
                ensure_ascii=False,
                allow_nan=False,
            )
        return json.dumps(
            {"status": "ok", "command": command, **result},
            ensure_ascii=False,
            allow_nan=False,
        )


def _err(msg: str) -> str:
    return json.dumps(
        {"status": "error", "error": msg}, ensure_ascii=False, allow_nan=False,
    )