"Vibe-Trading: Your Personal Trading Agent"
Python ai-agentalgorithmic-tradingbacktestingfintechllmmcpmulti-agentpythonquantitative-financetrading
"""Regression tests for DSML textual tool calls in the ReAct loop."""
from __future__ import annotations
import json
from pathlib import Path
from typing import Any, Callable
import pytest
from src.agent.loop import AgentLoop
from src.agent.progress import emit_progress
from src.agent.tools import BaseTool, ToolRegistry
from src.memory.persistent import PersistentMemory
from src.providers.chat import ChatLLM, LLMResponse, ToolCallRequest
class _Chunk:
"""Minimal LangChain AIMessageChunk stand-in."""
def __init__(self, content: str) -> None:
self.content = content
self.tool_calls: list[dict[str, Any]] = []
self.additional_kwargs: dict[str, Any] = {}
self.response_metadata = {"finish_reason": "stop"}
self.usage_metadata = None
def __add__(self, other: "_Chunk") -> "_Chunk":
return _Chunk(f"{self.content}{other.content}")
class _ScriptedStreamingLLM:
"""Return one scripted response per stream_chat call."""
def __init__(self, responses: list[str]) -> None:
self._responses = responses
def bind_tools(self, tools: list[dict[str, Any]]) -> "_ScriptedStreamingLLM":
return self
def stream(self, messages: list[dict[str, Any]], config: dict[str, Any] | None = None):
yield _Chunk(self._responses.pop(0))
class _EchoProbeTool(BaseTool):
"""Safe test tool proving DSML calls reach the normal tool executor."""
name = "echo_probe"
description = "Echo a marker for DSML tool-call regression tests."
parameters = {
"type": "object",
"properties": {"marker": {"type": "string"}},
"required": ["marker"],
}
repeatable = True
is_readonly = False
def execute(self, **kwargs: Any) -> str:
emit_progress("echoing", current=1, total=1)
return json.dumps({"status": "ok", "marker": kwargs.get("marker")})
def _chat_llm(fake_llm: _ScriptedStreamingLLM) -> ChatLLM:
client = ChatLLM.__new__(ChatLLM)
client.model_name = "deepseek-v4-pro"
client._llm = fake_llm
return client
def test_agent_loop_executes_dsml_textual_tool_call(
tmp_path: Path,
monkeypatch,
) -> None:
"""A pure DSML response must execute as a tool call instead of final text."""
class _ImmediateHeartbeatTimer:
def __init__(self, tool_name: str, interval: float, emit) -> None:
del interval
self._tool_name = tool_name
self._emit = emit
def __enter__(self):
self._emit({"tool": self._tool_name, "elapsed_s": 0.01})
return self
def __exit__(self, exc_type, exc, tb) -> None:
return None
monkeypatch.setattr(
"src.agent.tool_lifecycle.HeartbeatTimer", _ImmediateHeartbeatTimer
)
dsml = (
'<||DSML||tool_calls>'
'<||DSML||invoke name="echo_probe">'
'<||DSML||parameter name="marker" string="true">ran-dsml</||DSML||parameter>'
"</||DSML||invoke>"
"</||DSML||tool_calls>"
)
registry = ToolRegistry()
registry.register(_EchoProbeTool())
memory = PersistentMemory(memory_dir=tmp_path / "memory")
events: list[tuple[str, dict[str, Any]]] = []
agent = AgentLoop(
registry=registry,
llm=_chat_llm(_ScriptedStreamingLLM([dsml, "final answer"])),
event_callback=lambda event_type, payload: events.append((event_type, payload)),
max_iterations=2,
persistent_memory=memory,
)
agent.memory.run_dir = str(tmp_path / "run")
result = agent.run("use the probe")
assert result["status"] == "success"
assert result["content"] == "final answer"
tool_events = {
event_type: payload
for event_type, payload in events
if event_type
in {"tool_call", "tool_progress", "tool_heartbeat", "tool_result"}
}
assert set(tool_events) == {
"tool_call",
"tool_progress",
"tool_heartbeat",
"tool_result",
}
assert {
payload["call_id"] for payload in tool_events.values()
} == {"dsml_call_1"}
assert {
payload["tool"] for payload in tool_events.values()
} == {"echo_probe"}
def test_agent_loop_never_releases_tool_call_syntax_as_a_final_answer(
tmp_path: Path,
monkeypatch,
) -> None:
"""Forced-text final answers containing tool-call DSL are not released raw.
On the last iteration tools are withheld to guarantee a plain-text answer,
but a model can still emit its native tool-call markup as prose (plain or
fullwidth-vbar mojibake). That markup is not an answer: the loop must
retry when budget remains, otherwise release a deterministic message and
mark the run degraded instead of leaking ``<...tool_calls>`` to the user.
"""
class _ImmediateHeartbeatTimer:
def __init__(self, tool_name: str, interval: float, emit) -> None:
del interval
self._tool_name = tool_name
self._emit = emit
def __enter__(self):
self._emit({"tool": self._tool_name, "elapsed_s": 0.01})
return self
def __exit__(self, exc_type, exc, tb) -> None:
return None
monkeypatch.setattr(
"src.agent.tool_lifecycle.HeartbeatTimer", _ImmediateHeartbeatTimer
)
# The fullwidth-vbar (U+FF5C) form is what the real failure looked like; the
# streaming DSML parser does not recognize it, so it arrives as text.
garbage = "<││DSML││tool_calls><││invoke name=\"trading_quote\">"
registry = ToolRegistry()
events: list[tuple[str, dict[str, Any]]] = []
agent = AgentLoop(
registry=registry,
llm=_chat_llm(_ScriptedStreamingLLM([garbage, garbage])),
event_callback=lambda event_type, payload: events.append((event_type, payload)),
max_iterations=1, # the single iteration is the forced-text last one
persistent_memory=PersistentMemory(memory_dir=tmp_path / "memory"),
)
agent.memory.run_dir = str(tmp_path / "run")
result = agent.run("hello")
assert result["status"] == "success"
assert result.get("degraded") is True
assert result["iterations"] == 2
assert "tool-call syntax" in result["content"]
assert "<" not in result["content"]
assert not any(
event_type == "answer" and "<" in str(payload)
for event_type, payload in events
)
def test_closing_dsml_tags_alone_are_tool_call_syntax() -> None:
"""A DeepSeek draft once ended in closing tags only, inside its figures block;
the opening-tag pattern missed them and the gate got three unreadable lines."""
from src.agent.loop import _looks_like_tool_call_syntax
tail = "\n".join(
[
"```figures",
"0.3897 | observed | weight | rp",
"</||DSML||parameter>",
"</||DSML||invoke>",
"</||DSML||tool_calls>",
]
)
assert _looks_like_tool_call_syntax("Report.\n\n" + tail)
assert _looks_like_tool_call_syntax("done </||DSML||invoke>")
assert not _looks_like_tool_call_syntax("| 标的 | 权重 |\n|---|---|\n| 000001.SZ | 38.97% |")
@pytest.mark.parametrize("max_iterations", [1, 3])
def test_final_tool_syntax_gets_one_text_only_retry(
tmp_path: Path, max_iterations: int
) -> None:
"""A withheld final answer can be re-emitted after research hits its cap."""
dsml = (
'<||DSML||tool_calls><||DSML||invoke name="echo_probe">'
'<||DSML||parameter name="marker" string="true">ready</||DSML||parameter>'
"</||DSML||invoke></||DSML||tool_calls>"
)
garbage = '<││DSML││tool_calls><││invoke name="echo_probe">'
responses = [dsml] * (max_iterations - 1) + [garbage, "The report is ready."]
registry = ToolRegistry()
registry.register(_EchoProbeTool())
events: list[tuple[str, dict[str, Any]]] = []
agent = AgentLoop(
registry=registry,
llm=_chat_llm(_ScriptedStreamingLLM(responses)),
event_callback=lambda kind, payload: events.append((kind, payload)),
max_iterations=max_iterations,
persistent_memory=PersistentMemory(memory_dir=tmp_path / "memory"),
)
agent.memory.run_dir = str(tmp_path / "run")
result = agent.run("use the probe and report")
assert result["status"] == "success"
assert result["content"] == "The report is ready."
assert result["iterations"] == max_iterations + 1
assert not result.get("degraded")
assert sum(kind == "tool_result" for kind, _ in events) == max_iterations - 1
assert not responses
shown = ""
for kind, payload in events:
if kind == "stream_reset":
shown = ""
elif kind == "text_delta":
shown += payload["delta"]
assert shown == "The report is ready."
class _AnswerRetryLLM:
def __init__(self, responses: list[LLMResponse]) -> None:
self.responses = responses
self.offered_tools: list[list[dict[str, Any]] | None] = []
def stream_chat(
self,
messages: list[dict[str, Any]],
tools: list[dict[str, Any]] | None = None,
on_text_chunk: Callable[[str], None] | None = None,
**kwargs: Any,
) -> LLMResponse:
self.offered_tools.append(tools)
response = self.responses.pop(0)
if on_text_chunk and response.content:
on_text_chunk(response.content)
return response
def test_final_answer_retry_cannot_execute_unoffered_tools(tmp_path: Path) -> None:
"""A provider ignoring text-only mode cannot reopen tool execution."""
garbage = '<││DSML││tool_calls><││invoke name="echo_probe">'
llm = _AnswerRetryLLM(
[
LLMResponse(content=garbage),
LLMResponse(
content="",
tool_calls=[
ToolCallRequest(
id="retry-call",
name="echo_probe",
arguments={"marker": "unexpected"},
)
],
),
]
)
registry = ToolRegistry()
registry.register(_EchoProbeTool())
events: list[tuple[str, dict[str, Any]]] = []
agent = AgentLoop(
registry=registry,
llm=llm,
event_callback=lambda kind, payload: events.append((kind, payload)),
max_iterations=1,
persistent_memory=PersistentMemory(memory_dir=tmp_path / "memory"),
)
agent.memory.run_dir = str(tmp_path / "run")
result = agent.run("report")
assert not any(kind in {"tool_call", "tool_result"} for kind, _ in events)
assert llm.offered_tools == [None, None]
assert result["iterations"] == 2
assert result.get("degraded") is True
assert "tool-call syntax" in result["content"]
@pytest.mark.parametrize("token_budget, expected_calls", [(10, 1), (100, 2)])
def test_answer_retry_obeys_goal_budget_and_accounts_usage(
tmp_path: Path, monkeypatch, token_budget: int, expected_calls: int
) -> None:
from src.goal import GoalStore
monkeypatch.setenv("VIBE_TRADING_GOAL_DB_PATH", str(tmp_path / "goals.db"))
monkeypatch.setattr("src.agent.loop.SESSIONS_DIR", tmp_path / "sessions")
monkeypatch.setattr("src.agent.tuning.GOAL_MAX_CONTINUATIONS", 1, raising=False)
store = GoalStore()
goal = store.replace_goal(
session_id="retry-session",
objective="Write a report.",
criteria=["Deliver the report"],
token_budget=token_budget,
)
garbage = '<││DSML││tool_calls><││invoke name="echo_probe">'
llm = _AnswerRetryLLM(
[
LLMResponse(content=garbage, usage_metadata={"total_tokens": 15}),
LLMResponse(
content="The report is ready.", usage_metadata={"total_tokens": 20}
),
]
)
agent = AgentLoop(
registry=ToolRegistry(),
llm=llm,
max_iterations=1,
persistent_memory=PersistentMemory(memory_dir=tmp_path / "memory"),
)
agent.memory.run_dir = str(tmp_path / "run")
result = agent.run("Write a report.", session_id="retry-session")
assert len(llm.offered_tools) == expected_calls
updated = store.get_goal(goal.goal_id)
assert updated.tokens_used == (15 if expected_calls == 1 else 35)
assert updated.turns_used == 1
usage = json.loads((Path(result["run_dir"]) / "llm_usage.json").read_text())
assert usage["totals"]["calls"] == expected_calls
assert usage["totals"]["total_tokens"] == updated.tokens_used
if expected_calls == 1:
assert "budget" in result["reason"]
assert not result["content"]
else:
assert result["content"] == "The report is ready."
assert [row["iter"] for row in usage["per_iteration"]] == [1, 2]
def test_cancel_after_rejected_answer_prevents_retry(tmp_path: Path) -> None:
garbage = '<││DSML││tool_calls><││invoke name="echo_probe">'
llm = _AnswerRetryLLM([LLMResponse(content=garbage)])
agent = AgentLoop(
registry=ToolRegistry(),
llm=llm,
max_iterations=1,
persistent_memory=PersistentMemory(memory_dir=tmp_path / "memory"),
event_callback=lambda kind, payload: (
agent.cancel() if kind == "stream_reset" else None
),
)
agent.memory.run_dir = str(tmp_path / "run")
result = agent.run("Write a report.")
assert result["status"] == "cancelled"
assert len(llm.offered_tools) == 1
def test_retried_answer_still_passes_grounding(tmp_path: Path) -> None:
garbage = '<││DSML││tool_calls><││invoke name="echo_probe">'
llm = _AnswerRetryLLM(
[
LLMResponse(content=garbage),
LLMResponse(content="AAPL is trading at $999.00."),
]
)
agent = AgentLoop(
registry=ToolRegistry(),
llm=llm,
max_iterations=1,
persistent_memory=PersistentMemory(memory_dir=tmp_path / "memory"),
)
agent.memory.run_dir = str(tmp_path / "run")
result = agent.run("What is the AAPL stock price?")
assert result.get("degraded") is True
assert "$999.00" not in result["content"]
assert llm.offered_tools == [None, None]
def test_answer_retry_blocks_calls_after_grounding_correction(tmp_path: Path) -> None:
garbage = '<││DSML││tool_calls><││invoke name="echo_probe">'
llm = _AnswerRetryLLM(
[
LLMResponse(content="The return was 9.99%."),
LLMResponse(content=garbage),
LLMResponse(
content="",
tool_calls=[
ToolCallRequest(id="retry", name="echo_probe", arguments={})
],
),
]
)
registry = ToolRegistry()
registry.register(_EchoProbeTool())
events: list[tuple[str, dict[str, Any]]] = []
agent = AgentLoop(
registry=registry,
llm=llm,
max_iterations=2,
event_callback=lambda kind, payload: events.append((kind, payload)),
persistent_memory=PersistentMemory(memory_dir=tmp_path / "memory"),
)
agent.memory.run_dir = str(tmp_path / "run")
result = agent.run("Summarize the returns.")
assert result["iterations"] == 3
assert result.get("degraded") is True
assert "tool-call syntax" in result["content"]
assert not any(kind in {"tool_call", "tool_result"} for kind, _ in events)