"Vibe-Trading: Your Personal Trading Agent"
Python ai-agentalgorithmic-tradingbacktestingfintechllmmcpmulti-agentpythonquantitative-financetrading
Merge #1341 into #1352 and fold the cleared-marker representation change in
Haozhe Wu8a3acf7b1e9d
diff --cc agent/src/agent/loop.pyindex 8b033ad79,b6c52e99b..5814b4f1e--- a/agent/src/agent/loop.py+++ b/agent/src/agent/loop.py@@@ -63,13 -63,6 +63,12 @@@ COLLAPSE_TEXT_MIN = 240COLLAPSE_HEAD = 900COLLAPSE_TAIL = 500- # Placeholder contents that mean a tool result's real data was compacted- # away. Two sources: ``_microcompact`` clears old results in place to- # ``_CLEARED_CONTENT``, and ``_fix_tool_pairs`` inserts a ``_STUB_RESULT_CONTENT``- # stub for a call whose result a layer-3 fold consumed.- _CLEARED_CONTENT = "[cleared]"++# The stub ``_fix_tool_pairs`` inserts for a call whose result a layer-3 fold++# consumed. The other "data is gone" placeholder is layer 1's cleared marker,++# which is not a constant — it embeds the original payload length, so it is++# built by ``_cleared_text`` and matched by ``_is_cleared``.+_STUB_RESULT_CONTENT = "[Result from earlier context — see summary above]"+TAIL_TOKEN_BUDGET = 20_000SUMMARY_CHUNK_CHARS = 80_000@@@ -449,22 -469,28 +475,40 @@@ def _microcompact(messages: list) -> li"""tool_msgs = [m for m in messages if m.get("role") == "tool"]if len(tool_msgs) <= KEEP_RECENT:- return+ return []+ newly_cleared = []for msg in tool_msgs[:-KEEP_RECENT]:content = msg.get("content", "")- if isinstance(content, str) and len(content) > 100:- msg["content"] = _CLEARED_CONTENT+ # Skip a result already cleared: the marker is itself >100 chars, so+ # re-clearing it would rewrite the recorded original size with the+ # MARKER's length ("returned 287 characters") and re-report the tool+ # as newly unreadable on every later pass.+ if isinstance(content, str) and len(content) > 100 and not _is_cleared(content):+ # A bare "[cleared]" is indistinguishable from a tool that+ # returned nothing, so the model reports "no data was retrieved"+ # for data it did receive and this layer then deleted. Say which+ # it is, and say the result is recoverable.+ msg["content"] = _cleared_text(len(content))+ if msg.get("name"):+ newly_cleared.append(msg["name"])+ # Identified by prefix, not equality: the marker carries the original+ # length, so every cleared result is a different string.+ surviving = {m.get("name") for m in tool_msgs if not _is_cleared(m.get("content"))}+ return sorted(set(newly_cleared) - surviving)+def _result_data_gone(content: Any) -> bool:- """True when ``content`` is a placeholder meaning the tool result's real- data was compacted away (microcompact's ``[cleared]``, or the stub that- ``_fix_tool_pairs`` inserts after a layer-3 fold)."""- return isinstance(content, str) and (- content == _CLEARED_CONTENT or content == _STUB_RESULT_CONTENT- )++ """True when a tool result's real data is gone from context.++++ Two sources, and they must both be recognised: layer 1 overwrites an old++ result with the ``_CLEARED_PREFIX`` marker, and ``_fix_tool_pairs``++ inserts ``_STUB_RESULT_CONTENT`` for a call whose result a layer-3 fold++ consumed. The marker is matched by prefix, never equality — it embeds the++ original payload length, so no two cleared results are the same string.++ """++ return _is_cleared(content) or content == _STUB_RESULT_CONTENT++def _context_collapse(messages: list) -> None:"""Layer 2: fold long text blocks in older messages without LLM call.diff --cc agent/tests/test_loop_helpers.pyindex bcc8f3bf2,ba9a64332..cb7a8e7e9--- a/agent/tests/test_loop_helpers.py+++ b/agent/tests/test_loop_helpers.py@@@ -21,6 -21,6 +21,7 @@@ from src.agent.loop import_llm_timeout_seconds,_stall_timeout_seconds,_verification_ledger,++ _cleared_text,)@@@ -177,13 -257,13 +258,14 @@@ class TestContextCollapseassert msg["content"] == origdef test_skips_cleared_content(self) -> None:++ marker = _cleared_text(5000)messages = [{"role": "system", "content": "sys"}]for _ in range(COLLAPSE_PRESERVE_RECENT + 3):-- messages.append({"role": "tool", "content": "[cleared]"})++ messages.append({"role": "tool", "content": marker})_context_collapse(messages)-- # [cleared] should remain [cleared], not be collapsed++ # A cleared marker must survive layer 2 untouched.for m in messages[1:]:-- assert m["content"] == "[cleared]"++ assert m["content"] == markerdef test_no_op_when_too_few_messages(self) -> None:messages = [@@@ -600,128 -680,42 +682,171 @@@ def test_stall_timeout_seconds_default_assert _stall_timeout_seconds() > 0+# ---------------------------------------------------------------------------+# tool_calls[].function.arguments must count toward compaction sizing/relief+# ---------------------------------------------------------------------------++def test_tail_cut_index_counts_tool_call_arguments() -> None:+ """#tail-budget: a fat tool_call (empty content, huge arguments) must be+ pushed into the folded head, not counted as ~10 tokens in the tail."""+ from src.agent.loop import _tail_cut_index # new helper: absent on pre-fix code++ fat = "X" * 90_000 # ~22.5K tokens once arguments are counted+ body = [+ {"role": "user", "content": "old header"},+ {+ "role": "assistant",+ "content": "",+ "tool_calls": [+ {"id": "c1", "type": "function", "function": {"name": "t", "arguments": fat}}+ ],+ },+ ]+ # Old content-only sizing: both messages fit the 20K budget -> cut at 0.+ # New sizing: the call alone exceeds it -> cut at len(body), tail empty,+ # the oversized call goes into the folded head.+ assert _tail_cut_index(body) == len(body)+ assert _tail_cut_index(body) != 0+++def test_context_collapse_stubs_args_of_cleared_paired_call() -> None:+ """#layer-2: a tool_call whose paired result was [cleared] gets its huge+ arguments folded to a valid JSON '{}' stub (call id/name preserved)."""+ fat = "Y" * (COLLAPSE_TEXT_MIN * 3)+ call_msg = {+ "role": "assistant",+ "content": "",+ "tool_calls": [{"id": "c9", "type": "function", "function": {"name": "f", "arguments": fat}}],+ }+ msgs = [+ {"role": "system", "content": "sys"},+ {"role": "user", "content": "u0"},+ call_msg, # index 2 — inside the collapse window- {"role": "tool", "tool_call_id": "c9", "content": "[cleared]"},++ # The real layer-1 marker, not the old "[cleared]" literal: it embeds++ # the payload length, so _result_data_gone matches it by prefix.++ {"role": "tool", "tool_call_id": "c9", "content": _cleared_text(9000)},+ {"role": "assistant", "content": "a0"},+ {"role": "user", "content": "u1"},+ {"role": "assistant", "content": "a1"},+ {"role": "user", "content": "u2"},+ {"role": "assistant", "content": "a2"},+ {"role": "user", "content": "u3"},+ ]+ _context_collapse(msgs)+ assert call_msg["tool_calls"][0]["function"]["arguments"] == "{}"+ assert call_msg["tool_calls"][0]["function"]["name"] == "f"+++def test_context_collapse_keeps_args_when_result_intact_or_pending() -> None:+ """#layer-2 negative arm: arguments are NOT stubbed when the result still+ carries data, nor when the call is pending (no result in the transcript)."""+ fat = "Y" * (COLLAPSE_TEXT_MIN * 3)+ c1 = {+ "role": "assistant",+ "content": "",+ "tool_calls": [{"id": "c1", "type": "function", "function": {"name": "f", "arguments": fat}}],+ }+ c2 = {+ "role": "assistant",+ "content": "",+ "tool_calls": [{"id": "c2", "type": "function", "function": {"name": "g", "arguments": fat}}],+ }+ msgs = [+ {"role": "system", "content": "sys"},+ {"role": "user", "content": "u0"},+ c1, # index 2 — inside the collapse window+ {"role": "tool", "tool_call_id": "c1", "content": "still have the data"},+ c2, # pending — result never arrived+ {"role": "assistant", "content": "continue"},+ {"role": "user", "content": "u1"},+ {"role": "assistant", "content": "a1"},+ {"role": "user", "content": "u2"},+ {"role": "assistant", "content": "a2"},+ ]+ _context_collapse(msgs)+ assert c1["tool_calls"][0]["function"]["arguments"] == fat+ assert c2["tool_calls"][0]["function"]["arguments"] == fat+++def test_context_collapse_stubs_args_for_fix_tool_pairs_stub_result() -> None:+ """#adversarial: layer-3's ``_fix_tool_pairs`` stub result marker must also+ count as 'result data gone' — a call that survived a layer-3 fold gets its+ huge arguments stubbed too, not only microcompact's ``[cleared]``."""+ from src.agent.loop import _STUB_RESULT_CONTENT++ fat = "Y" * (COLLAPSE_TEXT_MIN * 3)+ call_msg = {+ "role": "assistant",+ "content": "",+ "tool_calls": [{"id": "c7", "type": "function", "function": {"name": "f", "arguments": fat}}],+ }+ msgs = [+ {"role": "system", "content": "sys"},+ {"role": "user", "content": "u0"},+ call_msg, # index 2 — inside the collapse window+ {"role": "tool", "tool_call_id": "c7", "content": _STUB_RESULT_CONTENT},+ {"role": "assistant", "content": "a0"},+ {"role": "user", "content": "u1"},+ {"role": "assistant", "content": "a1"},+ {"role": "user", "content": "u2"},+ {"role": "assistant", "content": "a2"},+ {"role": "user", "content": "u3"},+ ]+ _context_collapse(msgs)+ assert call_msg["tool_calls"][0]["function"]["arguments"] == "{}"+++def test_msg_estimate_chars_counts_dict_arguments() -> None:+ """#adversarial: dict (non-string) arguments must count toward sizing,+ consistent with ``estimate_tokens``' full serialization gate."""+ from src.agent.loop import _msg_estimate_chars++ big_dict = {"q": "X" * 4000}+ msg = {+ "role": "assistant",+ "content": "",+ "tool_calls": [{"function": {"name": "f", "arguments": big_dict}}],+ }+ assert _msg_estimate_chars(msg) >= 4000+ assert _msg_estimate_chars({"role": "user", "content": "hi"}) >= 2+++++ class TestMicrocompactMarkerIsStable:+ """The cleared-result marker is >100 chars, i.e. longer than the pruning+ threshold itself, so microcompact must not treat it as prunable payload."""++ def _tool_msgs(self, payload_len: int) -> list:+ msgs = [+ {+ "role": "tool",+ "tool_call_id": "c0",+ "name": "get_fund_flow",+ "content": "x" * payload_len,+ }+ ]+ msgs += [+ {"role": "tool", "tool_call_id": f"p{i}", "name": "pad", "content": "y" * 200}+ for i in range(KEEP_RECENT + 1)+ ]+ return msgs++ def test_marker_keeps_the_original_payload_length(self) -> None:+ payload_len = 4321+ messages = self._tool_msgs(payload_len)+ _microcompact(messages)+ first = messages[0]["content"]+ assert str(payload_len) in first, first++ # A second pass must not rewrite the marker with the marker's own+ # length, which would report a fabricated original size to the model.+ _microcompact(messages)+ assert messages[0]["content"] == first+ assert str(payload_len) in messages[0]["content"]++ def test_second_pass_reports_no_new_unreadable_tools(self) -> None:+ messages = self._tool_msgs(4321)+ assert _microcompact(messages) == ["get_fund_flow"]+ assert _microcompact(messages) == [], (+ "a tool already reported unreadable must not be re-reported, or the "+ "ledger is re-opened and traced on every later iteration"+ )