Skip to content

"Vibe-Trading: Your Personal Trading Agent"

13.4 KB
Raw Blame
"""Tests for the ``report_audit`` agent tool.

Covers the markdown data-point extraction (tables + ``label: value`` lines,
Chinese units), sampling (clamp + reproducibility), the verdict logic across
single/two-source cases (incl. the split-source WARN and the single-source
FAIL that the original upstream logic mishandled), the tool's JSON-Schema
contract, ``execute`` happy/error paths, and auto-discovery.
"""

from __future__ import annotations

import json
from typing import Any

import pytest

from src.tools import build_registry
from src.tools.report_audit_tool import (
    ReportAuditTool,
    _clean_num,
    _is_valid_label,
    _pct_diff,
    extract_data_points,
    render_verdict,
    sample_points,
)

# ── helpers ───────────────────────────────────────────────────────────────


def test_clean_num_handles_wide_comma() -> None:
    assert _clean_num("1,234") == 1234.0
    assert _clean_num("1,234") == 1234.0  # wide (full-width) comma
    assert _clean_num("abc") is None


def test_is_valid_label_filters_noise() -> None:
    assert _is_valid_label("营业收入") is True
    assert _is_valid_label("来源") is False     # skip-listed
    assert _is_valid_label("a") is False        # too short
    assert _is_valid_label("2024") is False     # year only
    assert _is_valid_label("+56%") is False     # bare growth rate


# ── extract_data_points ───────────────────────────────────────────────────


_REPORT_MD = (
    "收入:7518亿元\n"
    "毛利率:56%\n"
    "\n"
    "| 指标 | 2024 | 2023 |\n"
    "|------|------|------|\n"
    "| 营业收入 | 7518亿 | 6500亿 |\n"
    "| 净利润 | 1900亿 | 1600亿 |\n"
)


def test_extract_finds_table_and_kv_points() -> None:
    points = extract_data_points(_REPORT_MD)
    labels = {p["label"] for p in points}
    assert "收入" in labels              # KV line
    assert "毛利率" in labels            # KV line
    assert "营业收入 · 2024" in labels   # table cell
    assert "净利润 · 2024" in labels
    for p in points:
        assert {"id", "label", "reported_value", "unit", "line_number"} <= set(p)


def test_extract_assigns_unique_ids() -> None:
    ids = [p["id"] for p in extract_data_points(_REPORT_MD)]
    assert len(ids) == len(set(ids))


# ── sample_points ─────────────────────────────────────────────────────────


def _pts(n: int) -> list[dict[str, Any]]:
    return [
        {"id": i, "label": f"l{i}", "reported_value": float(i), "unit": "",
         "line_number": i, "raw_text": ""}
        for i in range(n)
    ]


def test_sample_returns_all_when_fewer_than_three() -> None:
    assert len(sample_points(_pts(2), ratio=0.5)) == 2


def test_sample_clamps_to_max_thirty() -> None:
    assert len(sample_points(_pts(500), ratio=1.0)) == 30


def test_sample_is_reproducible_with_seed() -> None:
    a = sample_points(_pts(20), ratio=0.5, seed=7)
    b = sample_points(_pts(20), ratio=0.5, seed=7)
    assert [p["id"] for p in a] == [p["id"] for p in b]


# ── _pct_diff ─────────────────────────────────────────────────────────────


def test_pct_diff() -> None:
    assert _pct_diff(100, 101) == 0.01
    assert _pct_diff(100, 100) == 0.0
    assert _pct_diff(0, 0) == 0.0
    assert _pct_diff(0, 5) == float("inf")


# ── render_verdict ────────────────────────────────────────────────────────


def test_verdict_single_source_pass() -> None:
    out = render_verdict([{
        "id": 1, "label": "rev", "reported_value": 100, "unit": "",
        "fetched_value": 100.5, "fetched_source": "m",
    }])
    assert out["verdict"] == "PASS"
    assert out["pass_count"] == 1 and out["fail_count"] == 0


def test_verdict_single_source_fail_fails_report() -> None:
    # Regression: a single-source failure must FAIL, not silently WARN.
    out = render_verdict([{
        "id": 1, "label": "rev", "reported_value": 100, "unit": "",
        "fetched_value": 150, "fetched_source": "m",
    }])
    assert out["verdict"] == "FAIL"
    assert out["fail_count"] == 1
    assert out["fail_items"][0]["label"] == "rev"


def test_verdict_two_sources_both_pass() -> None:
    out = render_verdict([{
        "id": 1, "label": "rev", "reported_value": 100, "unit": "",
        "fetched_value": 100.5, "fetched_source": "m",
        "fetched_value2": 99.8, "fetched_source2": "s",
    }])
    assert out["verdict"] == "PASS"
    assert out["pass_count"] == 1


def test_verdict_two_sources_both_fail() -> None:
    out = render_verdict([{
        "id": 1, "label": "rev", "reported_value": 100, "unit": "",
        "fetched_value": 150, "fetched_source": "m",
        "fetched_value2": 200, "fetched_source2": "s",
    }])
    assert out["verdict"] == "FAIL"
    assert out["fail_count"] == 1


def test_verdict_two_sources_split_is_warn_not_fail() -> None:
    # One source agrees, one misses -> caliber mismatch, not a hard fail.
    out = render_verdict([{
        "id": 1, "label": "rev", "reported_value": 100, "unit": "",
        "fetched_value": 100.5, "fetched_source": "m",
        "fetched_value2": 150, "fetched_source2": "s",
    }])
    assert out["verdict"] == "PASS"
    assert out["warn_count"] == 1 and out["fail_count"] == 0


def test_verdict_missing_reported_value_fails_against_nonzero_fetched() -> None:
    """A None reported value is unverifiable evidence, so the point fails."""
    out = render_verdict([{"reported_value": None, "fetched_value": 100.0, "label": "rev"}])
    assert out["verdict"] == "FAIL"
    assert out["fail_count"] == 1
    assert out["pass_count"] == 0


def test_verdict_missing_reported_value_fails_against_zero_fetched() -> None:
    """The false-PASS case: None vs fetched 0 must NOT be read as 0 == 0.

    Mapping a missing reported value to 0.0 makes ``_pct_diff(0, 0) == 0``, which
    would certify a report from evidence that was never verified.
    """
    out = render_verdict([{"reported_value": None, "fetched_value": 0.0, "label": "rev"}])
    assert out["verdict"] == "FAIL"
    assert out["fail_count"] == 1
    assert out["pass_count"] == 0
    assert "missing" in out["fail_items"][0]["reason"]
    assert out["fail_items"][0]["reported"] is None


def test_verdict_absent_reported_value_key_fails() -> None:
    """An entirely absent reported_value key is the same unverifiable case."""
    out = render_verdict([{"fetched_value": 0.0, "label": "rev"}])
    assert out["verdict"] == "FAIL"
    assert out["fail_count"] == 1


def test_verdict_genuine_reported_zero_still_passes() -> None:
    """A real reported 0 is valid data and must keep passing against fetched 0."""
    out = render_verdict([{"reported_value": 0, "fetched_value": 0.0, "label": "rev"}])
    assert out["verdict"] == "PASS"
    assert out["pass_count"] == 1
    assert out["fail_count"] == 0


@pytest.mark.parametrize("bad", [float("nan"), float("inf"), "n/a", {}])
def test_verdict_non_finite_reported_value_fails_with_reason(bad: Any) -> None:
    """NaN/Infinity/junk are equally unverifiable and stay JSON-serializable."""
    out = render_verdict([{"reported_value": bad, "fetched_value": 12.0, "label": "rev"}])
    assert out["verdict"] == "FAIL"
    assert "cannot verify" in out["fail_items"][0]["reason"]
    json.dumps(out, allow_nan=False)  # must not leak NaN/Infinity tokens


def test_execute_verdict_missing_reported_value_is_reported_not_certified() -> None:
    """End-to-end through the tool envelope: unverifiable evidence fails."""
    raw = ReportAuditTool().execute(
        command="verdict",
        results=[{"id": 1, "label": "rev", "reported_value": None,
                  "fetched_value": 0.0, "fetched_source": "m"}],
    )
    assert "NaN" not in raw and "Infinity" not in raw
    env = json.loads(raw)
    assert env["status"] == "ok"
    assert env["verdict"] == "FAIL"
    assert env["fail_items"][0]["reason"]


def test_verdict_skips_points_without_fetched_value() -> None:
    out = render_verdict([
        {"id": 1, "label": "a", "reported_value": 100, "fetched_value": None},
        {"id": 2, "label": "b", "reported_value": 100,
         "fetched_value": 100, "fetched_source": "m"},
    ])
    assert out["total"] == 1   # only the verified point counts
    assert out["verdict"] == "PASS"


def test_verdict_zero_verified_points_fails_closed() -> None:
    """A verdict where NOTHING was verified must not certify the report."""
    out = render_verdict([
        {"id": 1, "label": "a", "reported_value": 100, "fetched_value": None},
        {"id": 2, "label": "b", "reported_value": 200, "fetched_value": None},
    ])
    assert out["verdict"] == "FAIL"
    assert out["total"] == 0
    assert out["fail_count"] == 1
    assert "no data points were verified" in out["fail_items"][0]["reason"]


def test_verdict_garbage_fetched_value_fails_point() -> None:
    """A supplied-but-unusable fetched value fails the point, not a crash.

    Junk is NOT the same as an absent fetch: skipping it would let an agent
    drop a hard-to-verify number by padding it with a garbage fetch.
    """
    out = render_verdict([
        {"id": 1, "label": "a", "reported_value": 100,
         "fetched_value": "about 100", "fetched_source": "s"},
    ])
    assert out["verdict"] == "FAIL"
    assert out["total"] == 1
    assert out["fail_count"] == 1
    assert "fetched value is not a finite number" in out["fail_items"][0]["reason"]


def test_verdict_garbage_fetch_cannot_pad_a_clean_report() -> None:
    """One good point does not launder a garbage-fetch sibling."""
    out = render_verdict([
        {"id": 1, "label": "good", "reported_value": 100,
         "fetched_value": 100, "fetched_source": "m"},
        {"id": 2, "label": "hard", "reported_value": 500,
         "fetched_value": "see above", "fetched_source": "s"},
    ])
    assert out["verdict"] == "FAIL"
    assert out["total"] == 2
    assert out["fail_count"] == 1


def test_verdict_absent_fetched_with_good_point_still_passes() -> None:
    """Absent fetch is legitimately 'not verified' and stays skippable."""
    out = render_verdict([
        {"id": 1, "label": "a", "reported_value": 100, "fetched_value": None},
        {"id": 2, "label": "b", "reported_value": 100,
         "fetched_value": 100, "fetched_source": "m"},
    ])
    assert out["verdict"] == "PASS"
    assert out["total"] == 1


def test_verdict_garbage_second_source_falls_back_to_single() -> None:
    """A non-finite second source degrades to single-source, not a crash."""
    out = render_verdict([
        {"id": 1, "label": "a", "reported_value": 100,
         "fetched_value": 100, "fetched_source": "m",
         "fetched_value2": "n/a", "fetched_source2": "x"},
    ])
    assert out["verdict"] == "PASS"
    assert out["total"] == 1


# ── tool contract ─────────────────────────────────────────────────────


def test_tool_metadata() -> None:
    tool = ReportAuditTool()
    assert tool.name == "report_audit"
    assert tool.is_readonly is True
    assert tool.repeatable is True
    assert tool.parameters["required"] == ["command"]
    assert set(tool.parameters["properties"]["command"]["enum"]) == {"extract", "verdict"}


def test_tool_is_auto_discovered() -> None:
    assert "report_audit" in build_registry().tool_names


# ── execute ───────────────────────────────────────────────────────────────


def _run(**kwargs: Any) -> dict[str, Any]:
    return json.loads(ReportAuditTool().execute(**kwargs))


def test_execute_extract_happy() -> None:
    env = _run(command="extract", report_text=_REPORT_MD, ratio=0.5, seed=42)
    assert env["status"] == "ok"
    assert env["total_extracted"] >= 4
    assert env["sample_size"] >= 1
    assert "sample" in env and "hint" in env


def test_execute_verdict_fail() -> None:
    env = _run(command="verdict", results=[{
        "id": 1, "label": "rev", "reported_value": 100,
        "fetched_value": 150, "fetched_source": "m",
    }])
    assert env["status"] == "ok"
    assert env["verdict"] == "FAIL"


def test_execute_verdict_zero_reported_emits_strict_json() -> None:
    """reported=0 vs nonzero fetched must not emit bare Infinity tokens."""
    raw = ReportAuditTool().execute(
        command="verdict",
        results=[{
            "id": 1, "label": "zero", "reported_value": 0,
            "fetched_value": 5, "fetched_source": "m",
        }],
    )
    assert "Infinity" not in raw
    env = json.loads(raw)
    assert env["status"] == "ok"
    assert env["verdict"] == "FAIL"
    assert env["fail_items"][0]["diff1_pct"] is None
    json.dumps(env, allow_nan=False)


def test_execute_missing_report_text_is_error() -> None:
    assert _run(command="extract")["status"] == "error"


def test_execute_missing_results_is_error() -> None:
    assert _run(command="verdict")["status"] == "error"


def test_execute_unknown_command_is_error() -> None:
    env = _run(command="bogus")
    assert env["status"] == "error"
    assert "unknown command" in env["error"]