Skip to content

"Vibe-Trading: Your Personal Trading Agent"

23.0 KB
Raw Blame
"""Tests for the run-artifact reader tool.

Covers the contract of ``src.tools.run_artifact_tool``: manifest-driven
resolution (friendly aliases with and without a run card, manifest-listed
relative paths, tampered-manifest refusal), traversal rejection, CSV
coercion, offset paging, first+last-pinned downsampling, meta/JSON
envelopes including the manifest sha256, column projection, and the
byte-budget shrink loop.
"""

from __future__ import annotations

import csv
import hashlib
import json
from pathlib import Path

import pytest

from src.tools import run_artifact_tool
from src.tools.run_artifact_tool import RunArtifactTool, read_run_artifact

EQUITY_ROWS = 3821
EQUITY_COLUMNS = ["date", "equity", "cash", "drawdown"]
ROWS_ENVELOPE_KEYS = {
    "path",
    "artifact",
    "run_dir",
    "columns",
    "rows",
    "total_rows",
    "offset",
    "returned_rows",
    "truncated",
    "downsample",
    "next_offset",
}


def _expected_row(i: int) -> list:
    """Return the coerced form of equity.csv data row ``i``."""
    return [f"day{i:05d}", 100000 + i * 1.5, 1000 + i, None]


def _make_run(root: Path, rows: int = EQUITY_ROWS) -> Path:
    """Create a run directory carrying a programmatically generated equity.csv."""
    run_dir = root / "run"
    (run_dir / "artifacts").mkdir(parents=True, exist_ok=True)
    with (run_dir / "artifacts" / "equity.csv").open(
        "w", newline="", encoding="utf-8"
    ) as handle:
        writer = csv.writer(handle)
        writer.writerow(EQUITY_COLUMNS)
        for i in range(rows):
            writer.writerow([f"day{i:05d}", 100000 + i * 1.5, 1000 + i, ""])
    return run_dir


@pytest.fixture()
def run_dir(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path:
    """Return a run directory inside a tmp_path-scoped allowed run root."""
    monkeypatch.setenv("VIBE_TRADING_ALLOWED_RUN_ROOTS", str(tmp_path))
    return _make_run(tmp_path)


def _write_csv(path: Path, header: list[str], rows: list[list[str]]) -> None:
    """Write a small CSV artifact."""
    path.parent.mkdir(parents=True, exist_ok=True)
    with path.open("w", newline="", encoding="utf-8") as handle:
        writer = csv.writer(handle)
        writer.writerow(header)
        writer.writerows(rows)


def _manifest_entry(run_dir: Path, relative: str) -> dict:
    """Build a run_card manifest entry for a file on disk (run_card 1.0 shape)."""
    path = run_dir / relative
    return {
        "path": relative,
        "size_bytes": path.stat().st_size,
        "sha256": hashlib.sha256(path.read_bytes()).hexdigest(),
    }


def _write_run_card(run_dir: Path, entries: list[dict]) -> None:
    """Write a run_card.json carrying the given artifacts manifest."""
    (run_dir / "run_card.json").write_text(
        json.dumps({"schema_version": "1.0", "artifacts": entries}),
        encoding="utf-8",
    )


# --------------------------------------------------------------------------
# (a) downsample
# --------------------------------------------------------------------------


def test_downsample_caps_points_and_pins_first_last(run_dir: Path, monkeypatch) -> None:
    monkeypatch.setattr(run_artifact_tool, "_BYTE_BUDGET", 120_000)
    env = json.loads(
        read_run_artifact(str(run_dir), "equity", format="downsample", max_rows=1000)
    )
    assert set(env) == ROWS_ENVELOPE_KEYS
    assert len(env["rows"]) <= 1000
    assert env["rows"][0] == _expected_row(0)
    assert env["rows"][-1] == _expected_row(EQUITY_ROWS - 1)
    ds = env["downsample"]
    assert ds == {
        "stride": 4,
        "pinned_last": True,
        "algorithm": "every-nth",
        "total_rows": EQUITY_ROWS,
    }
    assert env["total_rows"] == EQUITY_ROWS
    assert env["truncated"] is False
    assert env["next_offset"] is None


def test_downsample_returns_everything_when_total_fits(run_dir: Path) -> None:
    _write_csv(run_dir / "artifacts" / "trades.csv", ["a"], [["1"], ["2"], ["3"]])
    env = json.loads(
        read_run_artifact(str(run_dir), "trades", format="downsample", max_rows=100)
    )
    assert env["rows"] == [[1], [2], [3]]
    assert env["downsample"]["stride"] == 1
    assert env["downsample"]["pinned_last"] is True


# --------------------------------------------------------------------------
# (b) offset paging walk
# --------------------------------------------------------------------------


def test_offset_paging_walk_is_lossless(run_dir: Path) -> None:
    gathered: list[list] = []
    offset = 0
    for _ in range(20):  # 3821 / 500 -> 8 pages; hard stop guards a loop bug
        env = json.loads(
            read_run_artifact(str(run_dir), "equity", offset=offset, max_rows=500)
        )
        assert env["offset"] == offset
        gathered.extend(env["rows"])
        if not env["truncated"]:
            assert env["next_offset"] is None
            break
        assert env["next_offset"] == offset + env["returned_rows"]
        offset = env["next_offset"]
    else:
        pytest.fail("paging walk did not terminate")
    assert len(gathered) == EQUITY_ROWS
    assert gathered == [_expected_row(i) for i in range(EQUITY_ROWS)]


def test_offset_past_end_returns_empty_page(run_dir: Path) -> None:
    env = json.loads(
        read_run_artifact(str(run_dir), "equity", offset=EQUITY_ROWS + 100)
    )
    assert env["rows"] == []
    assert env["returned_rows"] == 0
    assert env["truncated"] is False
    assert env["next_offset"] is None


def test_negative_offset_rejected(run_dir: Path) -> None:
    env = json.loads(read_run_artifact(str(run_dir), "equity", offset=-1))
    assert env["ok"] is False


def test_max_rows_clamped(run_dir: Path, monkeypatch: pytest.MonkeyPatch) -> None:
    monkeypatch.setattr(run_artifact_tool, "_BYTE_BUDGET", 10_000_000)
    _write_csv(
        run_dir / "artifacts" / "trades.csv", ["n"], [[str(i)] for i in range(5200)]
    )
    tiny = json.loads(read_run_artifact(str(run_dir), "trades", max_rows=0))
    assert tiny["returned_rows"] == 1
    huge = json.loads(read_run_artifact(str(run_dir), "trades", max_rows=10**6))
    assert huge["returned_rows"] == 5000
    assert huge["truncated"] is True
    assert huge["next_offset"] == 5000


# --------------------------------------------------------------------------
# (c) traversal / alias whitelist / manifest gate
# --------------------------------------------------------------------------


@pytest.mark.parametrize(
    "artifact",
    [
        "../../etc/passwd",
        "/etc/passwd",
        "ohlcv:../../x",
        "ohlcv:/abs",
        "ohlcv:",
        "ohlcv:.hidden",
        "equity.csv",
        "config",
        "",
    ],
)
def test_non_whitelisted_artifacts_rejected(run_dir: Path, artifact: str) -> None:
    env = json.loads(read_run_artifact(str(run_dir), artifact))
    assert env["ok"] is False
    assert set(env) == {"ok", "error", "hint"}
    assert env["hint"]


def test_run_dir_outside_allowed_roots_rejected(
    tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
    monkeypatch.setenv("VIBE_TRADING_ALLOWED_RUN_ROOTS", str(tmp_path))
    env = json.loads(read_run_artifact(str(tmp_path.parent / "elsewhere"), "equity"))
    assert env["ok"] is False
    assert "run roots" in env["error"]


def test_symlink_escape_rejected(run_dir: Path, tmp_path: Path) -> None:
    secret = tmp_path / "secret.csv"
    secret.write_text("a\n1\n", encoding="utf-8")
    (run_dir / "artifacts" / "trades.csv").symlink_to(secret)
    env = json.loads(read_run_artifact(str(run_dir), "trades"))
    assert env["ok"] is False


# --------------------------------------------------------------------------
# (c2) manifest-driven resolution
# --------------------------------------------------------------------------


def test_manifest_listed_json_artifact_resolves(run_dir: Path) -> None:
    payload = {"walk_forward": {"oos_sharpe": 0.9}}
    (run_dir / "artifacts" / "validation.json").write_text(
        json.dumps(payload), encoding="utf-8"
    )
    _write_run_card(run_dir, [_manifest_entry(run_dir, "artifacts/validation.json")])
    env = json.loads(read_run_artifact(str(run_dir), "artifacts/validation.json"))
    assert set(env) == {"artifact", "run_dir", "json", "size_bytes"}
    assert env["artifact"] == "artifacts/validation.json"
    assert env["json"] == payload


def test_manifest_listed_csv_artifact_resolves(run_dir: Path) -> None:
    _write_csv(
        run_dir / "artifacts" / "rebalance_notes.csv",
        ["date", "note"],
        [["2024-01-02", "held"]],
    )
    entry = _manifest_entry(run_dir, "artifacts/rebalance_notes.csv")
    _write_run_card(run_dir, [entry])
    env = json.loads(read_run_artifact(str(run_dir), "artifacts/rebalance_notes.csv"))
    assert env["columns"] == ["date", "note"]
    assert env["rows"] == [["2024-01-02", "held"]]
    meta = json.loads(
        read_run_artifact(str(run_dir), "artifacts/rebalance_notes.csv", format="meta")
    )
    assert meta["sha256"] == entry["sha256"]
    assert meta["manifest_size_bytes"] == entry["size_bytes"]


def test_relative_path_not_in_manifest_refused(run_dir: Path) -> None:
    _write_csv(run_dir / "artifacts" / "extra.csv", ["a"], [["1"]])
    _write_run_card(run_dir, [])  # a card exists but lists nothing
    env = json.loads(read_run_artifact(str(run_dir), "artifacts/extra.csv"))
    assert env["ok"] is False
    assert set(env) == {"ok", "error", "hint"}
    assert "manifest" in env["hint"]


@pytest.mark.parametrize("tampered", ["../evil.csv", "..\\evil.csv"])
def test_tampered_manifest_escape_entries_refused(
    run_dir: Path, tmp_path: Path, tampered: str
) -> None:
    (tmp_path / "evil.csv").write_text(
        "a\n1\n", encoding="utf-8"
    )  # the escape target exists
    _write_run_card(
        run_dir, [{"path": tampered, "size_bytes": 4, "sha256": "deadbeef"}]
    )
    env = json.loads(read_run_artifact(str(run_dir), tampered))
    assert env["ok"] is False
    assert set(env) == {"ok", "error", "hint"}


def test_tampered_manifest_absolute_path_refused(run_dir: Path, tmp_path: Path) -> None:
    secret = tmp_path / "secret.csv"
    secret.write_text("a\n1\n", encoding="utf-8")
    _write_run_card(
        run_dir, [{"path": str(secret), "size_bytes": 4, "sha256": "deadbeef"}]
    )
    env = json.loads(read_run_artifact(str(run_dir), str(secret)))
    assert env["ok"] is False


def test_manifest_symlink_entry_escaping_run_dir_refused(
    run_dir: Path, tmp_path: Path
) -> None:
    secret = tmp_path / "secret.csv"
    secret.write_text("a\n1\n", encoding="utf-8")
    (run_dir / "artifacts" / "link.csv").symlink_to(secret)
    _write_run_card(run_dir, [_manifest_entry(run_dir, "artifacts/link.csv")])
    env = json.loads(read_run_artifact(str(run_dir), "artifacts/link.csv"))
    assert env["ok"] is False


def test_manifest_listed_non_served_suffix_refused(run_dir: Path) -> None:
    (run_dir / "code").mkdir(parents=True, exist_ok=True)
    (run_dir / "code" / "signal_engine.py").write_text("print(1)\n", encoding="utf-8")
    _write_run_card(run_dir, [_manifest_entry(run_dir, "code/signal_engine.py")])
    env = json.loads(read_run_artifact(str(run_dir), "code/signal_engine.py"))
    assert env["ok"] is False
    assert "read_file" in env["hint"]


def test_aliases_work_without_run_card(run_dir: Path) -> None:
    assert not (run_dir / "run_card.json").exists()
    env = json.loads(read_run_artifact(str(run_dir), "equity", format="meta"))
    assert env["total_rows"] == EQUITY_ROWS
    assert "sha256" not in env and "manifest_size_bytes" not in env


def test_aliases_work_with_corrupt_run_card(run_dir: Path) -> None:
    (run_dir / "run_card.json").write_text("{oops not json", encoding="utf-8")
    env = json.loads(read_run_artifact(str(run_dir), "equity", format="meta"))
    assert env["total_rows"] == EQUITY_ROWS
    refused = json.loads(read_run_artifact(str(run_dir), "artifacts/equity.csv"))
    assert refused["ok"] is False
    assert "manifest" in refused["hint"]


def test_meta_exposes_manifest_sha256(run_dir: Path) -> None:
    entry = _manifest_entry(run_dir, "artifacts/equity.csv")
    _write_run_card(run_dir, [entry])
    env = json.loads(read_run_artifact(str(run_dir), "equity", format="meta"))
    assert env["sha256"] == entry["sha256"]
    assert env["manifest_size_bytes"] == entry["size_bytes"]
    assert env["size_bytes"] == (run_dir / "artifacts" / "equity.csv").stat().st_size


# --------------------------------------------------------------------------
# (d) structured output, no raw CSV
# --------------------------------------------------------------------------


def test_rows_are_structured_arrays_not_raw_csv(run_dir: Path) -> None:
    payload = read_run_artifact(str(run_dir), "equity", max_rows=50)
    assert "day00000,100000.0" not in payload  # no raw CSV line survives
    env = json.loads(payload)
    assert env["columns"] == EQUITY_COLUMNS
    for row in env["rows"]:
        assert isinstance(row, list)
        assert all(cell is None or isinstance(cell, (int, float, str)) for cell in row)
        assert all("\n" not in cell for cell in row if isinstance(cell, str))


# --------------------------------------------------------------------------
# (e) coercion
# --------------------------------------------------------------------------


def test_numeric_coercion(run_dir: Path) -> None:
    _write_csv(
        run_dir / "artifacts" / "metrics.csv",
        ["i", "f", "blank", "nan_col", "inf_col", "text"],
        [
            ["42", "3.14", "", "nan", "inf", "hello"],
            ["-7", "1e5", "   ", "NaN", "-inf", "1_000"],
            ["0", "2.0", "x", "Infinity", "-Infinity", "null"],
        ],
    )
    env = json.loads(read_run_artifact(str(run_dir), "metrics"))
    assert env["rows"][0] == [42, 3.14, None, None, None, "hello"]
    assert env["rows"][1] == [-7, 100000.0, None, None, None, "1_000"]
    assert env["rows"][2] == [0, 2.0, "x", None, None, "null"]


# --------------------------------------------------------------------------
# (f) meta
# --------------------------------------------------------------------------


def test_meta_format(run_dir: Path) -> None:
    env = json.loads(read_run_artifact(str(run_dir), "equity", format="meta"))
    assert set(env) == {"artifact", "run_dir", "path", "columns", "total_rows", "size_bytes"}
    assert env["artifact"] == "equity"
    assert env["columns"] == EQUITY_COLUMNS
    assert env["total_rows"] == EQUITY_ROWS
    assert env["size_bytes"] == (run_dir / "artifacts" / "equity.csv").stat().st_size


def test_unknown_format_rejected(run_dir: Path) -> None:
    env = json.loads(read_run_artifact(str(run_dir), "equity", format="yaml"))
    assert env["ok"] is False


# --------------------------------------------------------------------------
# (g) projection
# --------------------------------------------------------------------------


def test_columns_projection(run_dir: Path) -> None:
    env = json.loads(
        read_run_artifact(
            str(run_dir), "equity", columns=["equity", "date"], max_rows=10
        )
    )
    assert env["columns"] == ["equity", "date"]
    assert env["rows"][0] == [100000.0, "day00000"]
    assert all(len(row) == 2 for row in env["rows"])


def test_unknown_column_lists_valid_columns(run_dir: Path) -> None:
    env = json.loads(
        read_run_artifact(str(run_dir), "equity", columns=["equity", "nope"])
    )
    assert env["ok"] is False
    assert "nope" in env["error"]
    for column in EQUITY_COLUMNS:
        assert column in env["error"]


# --------------------------------------------------------------------------
# (h) byte budget
# --------------------------------------------------------------------------


def test_byte_budget_shrinks_rows_page_honestly(
    run_dir: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
    monkeypatch.setattr(run_artifact_tool, "_BYTE_BUDGET", 3000)
    payload = read_run_artifact(str(run_dir), "equity", max_rows=1000)
    assert len(payload) <= 3000
    env = json.loads(payload)  # never broken JSON
    assert 0 < env["returned_rows"] < 1000
    assert env["returned_rows"] == len(env["rows"])
    assert env["truncated"] is True
    assert env["next_offset"] == env["offset"] + env["returned_rows"]
    assert env["rows"][0] == _expected_row(0)
    # The advertised resume point serves a valid next page.
    follow = json.loads(
        read_run_artifact(
            str(run_dir), "equity", offset=env["next_offset"], max_rows=1000
        )
    )
    assert follow["rows"][0] == _expected_row(env["next_offset"])


def test_byte_budget_restrides_downsample(
    run_dir: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
    monkeypatch.setattr(run_artifact_tool, "_BYTE_BUDGET", 3000)
    payload = read_run_artifact(
        str(run_dir), "equity", format="downsample", max_rows=1000
    )
    assert len(payload) <= 3000
    env = json.loads(payload)
    assert env["returned_rows"] < 1000
    assert env["downsample"]["stride"] > 4  # re-striden, not silently truncated
    assert env["downsample"]["total_rows"] == EQUITY_ROWS
    assert env["rows"][0] == _expected_row(0)
    assert env["rows"][-1] == _expected_row(EQUITY_ROWS - 1)


def test_byte_budget_never_breaks_json_at_floor(
    run_dir: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
    monkeypatch.setattr(run_artifact_tool, "_BYTE_BUDGET", 10)
    env = json.loads(read_run_artifact(str(run_dir), "equity", max_rows=100))
    assert env["ok"] is False
    assert "budget" in env["error"]


# --------------------------------------------------------------------------
# (i) ohlcv
# --------------------------------------------------------------------------


def test_ohlcv_code_maps_to_artifacts_file(run_dir: Path) -> None:
    _write_csv(
        run_dir / "artifacts" / "ohlcv_600519.SH.csv",
        ["date", "open", "close"],
        [["2024-01-02", "100.5", "101.25"]],
    )
    env = json.loads(read_run_artifact(str(run_dir), "ohlcv:600519.SH"))
    assert env["artifact"] == "ohlcv:600519.SH"
    assert env["columns"] == ["date", "open", "close"]
    assert env["rows"] == [["2024-01-02", 100.5, 101.25]]


# --------------------------------------------------------------------------
# (j) run_card / corrupt JSON
# --------------------------------------------------------------------------


def test_run_card_returns_parsed_json_regardless_of_format(run_dir: Path) -> None:
    card = {"run_id": "abc", "metrics": {"sharpe": 1.25}, "tags": ["x"]}
    (run_dir / "run_card.json").write_text(json.dumps(card), encoding="utf-8")
    for fmt in ("rows", "downsample", "meta"):
        env = json.loads(read_run_artifact(str(run_dir), "run_card", format=fmt))
        assert set(env) == {"artifact", "run_dir", "json", "size_bytes"}
        assert env["json"] == card
        assert env["size_bytes"] == (run_dir / "run_card.json").stat().st_size


def test_corrupt_run_card_rejected(run_dir: Path) -> None:
    (run_dir / "run_card.json").write_text("{oops not json", encoding="utf-8")
    env = json.loads(read_run_artifact(str(run_dir), "run_card"))
    assert env["ok"] is False
    assert set(env) == {"ok", "error", "hint"}


# --------------------------------------------------------------------------
# (k) missing file
# --------------------------------------------------------------------------


def test_missing_artifact_has_hint(run_dir: Path) -> None:
    env = json.loads(read_run_artifact(str(run_dir), "trades"))
    assert env["ok"] is False
    assert env["hint"]
    assert "not found" in env["error"]


def test_manifest_listed_but_missing_on_disk_has_hint(run_dir: Path) -> None:
    _write_run_card(
        run_dir,
        [{"path": "artifacts/validation.json", "size_bytes": 2, "sha256": "deadbeef"}],
    )
    env = json.loads(read_run_artifact(str(run_dir), "artifacts/validation.json"))
    assert env["ok"] is False
    assert "not found" in env["error"]


# --------------------------------------------------------------------------
# envelope hygiene + tool class
# --------------------------------------------------------------------------


def test_success_envelope_carries_no_ok_field(run_dir: Path) -> None:
    env = json.loads(read_run_artifact(str(run_dir), "equity", max_rows=5))
    assert "ok" not in env and "status" not in env
    assert set(env) == ROWS_ENVELOPE_KEYS
    assert env["downsample"] is None
    assert env["run_dir"] == str(run_dir)


def test_tool_class_contract(run_dir: Path) -> None:
    tool = RunArtifactTool()
    assert tool.name == "read_run_artifact"
    assert tool.is_readonly is True
    assert tool.repeatable is True
    assert tool.description
    assert "progress" not in tool.description
    schema = tool.to_openai_schema()
    props = schema["function"]["parameters"]["properties"]
    assert props["format"]["enum"] == ["rows", "downsample", "meta"]
    assert schema["function"]["parameters"]["required"] == ["run_dir", "artifact"]

    meta = json.loads(
        tool.execute(run_dir=str(run_dir), artifact="equity", format="meta")
    )
    assert meta["total_rows"] == EQUITY_ROWS
    default = json.loads(
        tool.execute(run_dir=str(run_dir), artifact="equity", max_rows=3)
    )
    assert set(default) == ROWS_ENVELOPE_KEYS
    assert default["returned_rows"] == 3


def test_default_page_survives_actual_agent_delivery_cap(run_dir):
    from src.config.limits import TOOL_RESULT_LIMIT, truncate_tool_result
    payload = read_run_artifact(str(run_dir), "equity")
    assert len(payload) <= TOOL_RESULT_LIMIT
    assert truncate_tool_result(payload) == payload
    first = json.loads(payload)
    assert first["path"] == str(run_dir / "artifacts/equity.csv")
    next_page = json.loads(read_run_artifact(str(run_dir), "equity", offset=first["next_offset"]))
    assert next_page["rows"][0] == _expected_row(first["returned_rows"])


def test_oversized_json_is_an_actionable_error(run_dir):
    from src.config.limits import TOOL_RESULT_LIMIT
    (run_dir / "run_card.json").write_text(json.dumps({"body": "x" * TOOL_RESULT_LIMIT}))
    payload = read_run_artifact(str(run_dir), "run_card")
    assert len(payload) <= TOOL_RESULT_LIMIT
    assert json.loads(payload)["ok"] is False


def test_oversized_row_requires_projection(run_dir):
    _write_csv(run_dir / "artifacts/trades.csv", ["code", "note"], [["000001", "x" * 20000]])
    assert json.loads(read_run_artifact(str(run_dir), "trades"))["ok"] is False
    projected = json.loads(read_run_artifact(str(run_dir), "trades", columns=["code"]))
    assert projected["rows"] == [["000001"]]


def test_shape_and_page_do_not_coerce_unselected_rows(run_dir, monkeypatch):
    original = run_artifact_tool._coerce_value
    count = 0
    def coercion(value):
        nonlocal count
        count += 1
        return original(value)
    monkeypatch.setattr(run_artifact_tool, "_coerce_value", coercion)
    read_run_artifact(str(run_dir), "equity", format="meta")
    assert count == 0
    read_run_artifact(str(run_dir), "equity", max_rows=3)
    assert count == 3 * len(EQUITY_COLUMNS)



def test_oversized_json_meta_reports_size_without_content(run_dir):
    from src.config.limits import TOOL_RESULT_LIMIT
    card = run_dir / "run_card.json"
    card.write_text(json.dumps({"body": "x" * TOOL_RESULT_LIMIT}))
    result = json.loads(read_run_artifact(str(run_dir), "run_card", format="meta"))
    assert result["size_bytes"] == card.stat().st_size
    assert result["content_omitted"] is True
    assert result["format"] == "json"
    assert "json" not in result