Open-source AI penetration testing tool to find and fix your app’s vulnerabilities.
Python agentsai-hackingai-penetration-testingai-pentestingai-securityartificial-intelligencebug-bountycode-qualityctf-toolscybersecuritycybersecurity-toolsethical-hackinghackingllm-securityoffensive-securitypenetration-testingpentesting-toolsred-teamingsecuritysecurity-automation
fix(llm): keep tool-call ids unique so a recycled id can't erase history (#976)
devin-ai-integration[bot]68ea6fca6531
diff --git a/pyproject.toml b/pyproject.tomlindex 38d5154c..d264dcca 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -236,6 +236,7 @@ ignore = ["strix/interface/auth_cli.py" = ["N802"]"tests/test_codex_streaming.py" = ["N802"]"tests/test_disable_streaming.py" = ["N802"]+"tests/test_tool_call_ids.py" = ["N802"]"tests/test_unknown_tool_recovery.py" = ["N802"]"tests/test_report_pdf.py" = ["S105", "S106"]# Stdlib HTTP handler overrides (do_GET/do_POST) and lazy imports that avoid adiff --git a/strix/config/models.py b/strix/config/models.pyindex a3bb481a..8685a2fb 100644--- a/strix/config/models.py+++ b/strix/config/models.py@@ -6,7 +6,7 @@ import contextlibimport inspectimport osimport time-from typing import TYPE_CHECKING, Any+from typing import TYPE_CHECKING, Any, castfrom agents import (set_default_openai_api,@@ -24,12 +24,18 @@ from agents.retry import (RetryPolicyContext,retry_policies,)-from openai.types.responses import Response, ResponseCompletedEvent+from openai.types.responses import (+ Response,+ ResponseCompletedEvent,+ ResponseOutputItemAddedEvent,+ ResponseOutputItemDoneEvent,+)from openai.types.responses.response_usage import ResponseUsagefrom openai.types.shared import Reasoningfrom strix.config import codexfrom strix.config.loader import load_settings+from strix.config.tool_call_ids import TurnCallIdRewriter, dedupe_inputif TYPE_CHECKING:@@ -229,6 +235,105 @@ class _NonStreamingModel(Model):yield _completed_stream_event(response, getattr(self._inner, "model", None))+class _UniqueToolCallIdModel(Model):+ """Keep tool-call ids unique so a recycled id can't invalidate the history.++ Providers that number tool calls per turn (``exec_command:0``, ...) restart+ the counter each turn, so the same id eventually appears twice in one+ conversation and strict providers reject every subsequent request. Ids that+ collide with the history are rewritten before the turn is recorded, and+ already-corrupted histories are repaired on the way out.+ """++ def __init__(self, inner: Model) -> None:+ self._inner = inner++ async def close(self) -> None:+ await self._inner.close()++ def get_retry_advice(self, request: ModelRetryAdviceRequest) -> ModelRetryAdvice | None:+ return self._inner.get_retry_advice(request)++ async def get_response(+ self,+ system_instructions: str | None,+ input: str | list[TResponseInputItem], # noqa: A002+ model_settings: ModelSettings,+ tools: list[Tool],+ output_schema: AgentOutputSchemaBase | None,+ handoffs: list[Handoff],+ tracing: ModelTracing,+ *,+ previous_response_id: str | None,+ conversation_id: str | None,+ prompt: ResponsePromptParam | None,+ ) -> ModelResponse:+ sanitized = dedupe_input(input)+ rewriter = TurnCallIdRewriter(sanitized)+ response = await self._inner.get_response(+ system_instructions,+ cast("str | list[TResponseInputItem]", sanitized),+ model_settings,+ tools,+ output_schema,+ handoffs,+ tracing,+ previous_response_id=previous_response_id,+ conversation_id=conversation_id,+ prompt=prompt,+ )+ response.output = rewriter.rewrite_items(list(response.output))+ return response++ async def stream_response(+ self,+ system_instructions: str | None,+ input: str | list[TResponseInputItem], # noqa: A002+ model_settings: ModelSettings,+ tools: list[Tool],+ output_schema: AgentOutputSchemaBase | None,+ handoffs: list[Handoff],+ tracing: ModelTracing,+ *,+ previous_response_id: str | None,+ conversation_id: str | None,+ prompt: ResponsePromptParam | None,+ ) -> AsyncIterator[TResponseStreamEvent]:+ sanitized = dedupe_input(input)+ rewriter = TurnCallIdRewriter(sanitized)+ stream = self._inner.stream_response(+ system_instructions,+ cast("str | list[TResponseInputItem]", sanitized),+ model_settings,+ tools,+ output_schema,+ handoffs,+ tracing,+ previous_response_id=previous_response_id,+ conversation_id=conversation_id,+ prompt=prompt,+ )+ async for event in stream:+ yield _rewrite_event_call_ids(event, rewriter)+++def _rewrite_event_call_ids(+ event: TResponseStreamEvent, rewriter: TurnCallIdRewriter+) -> TResponseStreamEvent:+ if isinstance(event, ResponseOutputItemAddedEvent | ResponseOutputItemDoneEvent):+ rewritten = rewriter.rewrite_item(event.item)+ if rewritten is not event.item:+ return event.model_copy(update={"item": rewritten})+ return event+ if isinstance(event, ResponseCompletedEvent):+ output = rewriter.rewrite_items(list(event.response.output))+ if output != list(event.response.output):+ return event.model_copy(+ update={"response": event.response.model_copy(update={"output": output})}+ )+ return event++def _completed_stream_event(model_response: ModelResponse, model_name: object | None) -> TResponseStreamEvent:@@ -305,8 +410,8 @@ class StrixProvider(MultiProvider):)model = super().get_model(model_name)if llm.disable_streaming:- return _NonStreamingModel(model)- return model+ model = _NonStreamingModel(model)+ return _UniqueToolCallIdModel(model)DEFAULT_MODEL_RETRY = ModelRetrySettings(diff --git a/strix/config/tool_call_ids.py b/strix/config/tool_call_ids.pynew file mode 100644index 00000000..31f78850--- /dev/null+++ b/strix/config/tool_call_ids.py@@ -0,0 +1,117 @@+"""Keep tool-call ids unique within a conversation.++Some providers return per-turn tool-call ids (``exec_command:0``,+``exec_command:1``, ...) whose counter restarts on every turn. Once the same+id appears twice in one conversation, the request payload has two assistant+tool calls sharing an id and strict providers reject the whole turn, which+permanently kills the agent because the malformed history is replayed on+every retry. Rewriting duplicates to fresh unique ids keeps the history+valid for any provider.+"""++from __future__ import annotations++from collections import defaultdict, deque+from typing import Any+from uuid import uuid4++from openai.types.responses import ResponseFunctionToolCall+++def new_call_id() -> str:+ return f"call_{uuid4().hex}"+++def collect_call_ids(items: list[Any]) -> set[str]:+ used: set[str] = set()+ for item in items:+ if isinstance(item, dict):+ call_id = item.get("call_id")+ if isinstance(call_id, str):+ used.add(call_id)+ elif isinstance(item, ResponseFunctionToolCall):+ used.add(item.call_id)+ return used+++def dedupe_history_call_ids(items: list[Any]) -> tuple[list[Any], bool]:+ """Rewrite duplicate call ids in a conversation history.++ Outputs are paired with their call by order, so parallel calls that share+ an id keep answering the right call after the rewrite.+ """+ used: set[str] = set()+ pending: dict[str, deque[str]] = defaultdict(deque)+ rebuilt: list[Any] = []+ changed = False++ for item in items:+ if not isinstance(item, dict):+ rebuilt.append(item)+ continue+ call_id = item.get("call_id")+ if not isinstance(call_id, str):+ rebuilt.append(item)+ continue++ kind = item.get("type")+ if kind == "function_call":+ effective = call_id+ if call_id in used:+ effective = new_call_id()+ item = {**item, "call_id": effective} # noqa: PLW2901+ changed = True+ used.add(effective)+ pending[call_id].append(effective)+ elif kind == "function_call_output":+ queue = pending.get(call_id)+ if queue:+ effective = queue.popleft()+ if effective != call_id:+ item = {**item, "call_id": effective} # noqa: PLW2901+ changed = True+ rebuilt.append(item)++ return rebuilt, changed+++def dedupe_input(model_input: str | list[Any]) -> str | list[Any]:+ if isinstance(model_input, str):+ return model_input+ rebuilt, changed = dedupe_history_call_ids(model_input)+ return rebuilt if changed else model_input+++class TurnCallIdRewriter:+ """Rewrite a single turn's tool-call ids that collide with the history.++ A turn's items surface several times (streamed item events, then the+ completed response), so the same original id must always map to the same+ replacement within the turn.+ """++ def __init__(self, model_input: str | list[Any]) -> None:+ self._used = set() if isinstance(model_input, str) else collect_call_ids(model_input)+ self._remap: dict[str, str] = {}+ self._settled: set[str] = set()++ def rewrite_item(self, item: Any) -> Any:+ if not isinstance(item, ResponseFunctionToolCall):+ return item+ original = item.call_id+ if original in self._settled:+ return item+ replacement = self._remap.get(original)+ if replacement is None:+ if original not in self._used:+ self._used.add(original)+ self._settled.add(original)+ return item+ replacement = new_call_id()+ self._remap[original] = replacement+ self._used.add(replacement)+ self._settled.add(replacement)+ return item.model_copy(update={"call_id": replacement})++ def rewrite_items(self, items: list[Any]) -> list[Any]:+ return [self.rewrite_item(item) for item in items]diff --git a/tests/test_disable_streaming.py b/tests/test_disable_streaming.pyindex 1d667e61..d8438a54 100644--- a/tests/test_disable_streaming.py+++ b/tests/test_disable_streaming.py@@ -31,7 +31,7 @@ from openai.types.responses import (from strix.config import codex, loaderfrom strix.config.loader import load_settings-from strix.config.models import StrixProvider, _NonStreamingModel+from strix.config.models import StrixProvider, _NonStreamingModel, _UniqueToolCallIdModelif TYPE_CHECKING:@@ -299,10 +299,11 @@ def test_get_model_wraps_when_disabled(load_settings()model = StrixProvider().get_model("openai/gpt-4o-mini")- assert isinstance(model, _NonStreamingModel)+ assert isinstance(model, _UniqueToolCallIdModel)+ assert isinstance(model._inner, _NonStreamingModel)-def test_get_model_unwrapped_by_default(+def test_get_model_keeps_streaming_by_default(monkeypatch: pytest.MonkeyPatch, _reset_settings: None) -> None:inner = _DummyModel()@@ -310,7 +311,8 @@ def test_get_model_unwrapped_by_default(load_settings()model = StrixProvider().get_model("openai/gpt-4o-mini")- assert model is inner+ assert isinstance(model, _UniqueToolCallIdModel)+ assert model._inner is innerdef test_get_model_does_not_wrap_subscription_model(diff --git a/tests/test_tool_call_ids.py b/tests/test_tool_call_ids.pynew file mode 100644index 00000000..a4033f4b--- /dev/null+++ b/tests/test_tool_call_ids.py@@ -0,0 +1,249 @@+"""Tests for tool-call id uniqueness.++Providers that number tool calls per turn (``exec_command:0``, ``:1``, ...)+restart the counter on every turn, so the same id eventually appears twice in+one conversation. Strict providers then reject the whole request, and because+the history is replayed on every retry the agent can never recover. A gateway+that validates id uniqueness the way those providers do proves both the+failure and the fix.+"""++from __future__ import annotations++import json+import threading+from http.server import BaseHTTPRequestHandler, HTTPServer+from typing import TYPE_CHECKING, Any++import pytest+from agents import Agent, Runner, function_tool+from agents.models.interface import Model, ModelProvider+from agents.models.openai_chatcompletions import OpenAIChatCompletionsModel+from agents.run import RunConfig+from openai import AsyncOpenAI+from openai.types.responses import ResponseFunctionToolCall++from strix.config.models import _NonStreamingModel, _UniqueToolCallIdModel+from strix.config.tool_call_ids import TurnCallIdRewriter, dedupe_history_call_ids+++if TYPE_CHECKING:+ from collections.abc import Iterator+++def _tool_call_completion(call_id: str, n: int = 1) -> dict[str, Any]:+ return {+ "id": "chatcmpl-1",+ "object": "chat.completion",+ "created": 0,+ "model": "gw-model",+ "choices": [+ {+ "index": 0,+ "finish_reason": "tool_calls",+ "message": {+ "role": "assistant",+ "content": None,+ "tool_calls": [+ {+ "id": call_id,+ "type": "function",+ "function": {"name": "do_thing", "arguments": json.dumps({"n": n})},+ }+ ],+ },+ }+ ],+ "usage": {"prompt_tokens": 5, "completion_tokens": 2, "total_tokens": 7},+ }+++def _text_completion(text: str) -> dict[str, Any]:+ return {+ "id": "chatcmpl-2",+ "object": "chat.completion",+ "created": 0,+ "model": "gw-model",+ "choices": [+ {"index": 0, "finish_reason": "stop", "message": {"role": "assistant", "content": text}}+ ],+ "usage": {"prompt_tokens": 5, "completion_tokens": 3, "total_tokens": 8},+ }+++_REQUESTS: list[list[dict[str, Any]]] = []+++def _assistant_call_ids(messages: list[dict[str, Any]]) -> list[str]:+ return [str(call.get("id")) for message in messages for call in message.get("tool_calls") or []]+++def _tool_results(messages: list[dict[str, Any]]) -> list[str]:+ return [str(m.get("content")) for m in messages if m.get("role") == "tool"]+++class _StrictHandler(BaseHTTPRequestHandler):+ """Gateway that rejects a history reusing a tool-call id, like strict providers do."""++ def log_message(self, *args: Any) -> None:+ pass++ def do_POST(self) -> None:+ length = int(self.headers.get("Content-Length", 0))+ body = json.loads(self.rfile.read(length) or b"{}")+ messages = body.get("messages", [])+ _REQUESTS.append(messages)+ call_ids = _assistant_call_ids(messages)++ if len(call_ids) != len(set(call_ids)):+ self._respond(+ 400,+ {+ "error": {+ "message": (+ "tool messages need a resolvable tool name: carry `tool`/`name`, "+ "or match a preceding assistant tool_call by order"+ )+ }+ },+ )+ return++ turn = len(_REQUESTS)+ if turn <= 2:+ # The provider restarts its per-turn counter, so both turns say ":0".+ self._respond(200, _tool_call_completion("exec_command:0", n=turn))+ else:+ self._respond(200, _text_completion("all done"))++ def _respond(self, status: int, payload: dict[str, Any]) -> None:+ encoded = json.dumps(payload).encode()+ self.send_response(status)+ self.send_header("Content-Type", "application/json")+ self.send_header("Content-Length", str(len(encoded)))+ self.end_headers()+ self.wfile.write(encoded)+++def strict_gateway() -> Iterator[str]:+ _REQUESTS.clear()+ server = HTTPServer(("127.0.0.1", 0), _StrictHandler)+ thread = threading.Thread(target=server.serve_forever, daemon=True)+ thread.start()+ try:+ yield f"http://127.0.0.1:{server.server_address[1]}/v1"+ finally:+ server.shutdown()+ server.server_close()+++def _model(base_url: str) -> Model:+ # The gateway answers plain JSON, so the run loop's streamed turns are+ # served non-streamed; the ids on the wire are the same either way.+ client = AsyncOpenAI(api_key="tok", base_url=base_url, max_retries=0)+ return _NonStreamingModel(OpenAIChatCompletionsModel(model="gw-model", openai_client=client))+++async def _run_agent(base_url: str, *, wrap: bool) -> Any:+ @function_tool+ def do_thing(n: int) -> str:+ return f"did {n}"++ class _Provider(ModelProvider):+ def get_model(self, model_name: str | None) -> Model: # noqa: ARG002+ model = _model(base_url)+ return _UniqueToolCallIdModel(model) if wrap else model++ agent = Agent(name="t", instructions="use the tool", tools=[do_thing], model="gw-model")+ result = Runner.run_streamed(+ agent, input="please", run_config=RunConfig(model_provider=_Provider())+ )+ async for _ in result.stream_events():+ pass+ return result+++async def test_recycled_call_id_erases_a_turn_without_the_wrapper(strict_gateway: str) -> None:+ # Repro: two turns run a tool and both are labelled ``exec_command:0``, so+ # the colliding call and its result are dropped as duplicates. The agent+ # ends the run having silently lost a turn of its own work — and a provider+ # that does not drop them instead rejects the malformed history outright.+ result = await _run_agent(strict_gateway, wrap=False)++ assert result.final_output == "all done"+ assert _assistant_call_ids(_REQUESTS[-1]) == ["exec_command:0"]+ assert _tool_results(_REQUESTS[-1]) == ["did 2"]+++async def test_recycled_call_id_is_rewritten_so_no_turn_is_lost(strict_gateway: str) -> None:+ result = await _run_agent(strict_gateway, wrap=True)++ assert result.final_output == "all done"+ call_ids = _assistant_call_ids(_REQUESTS[-1])+ assert len(call_ids) == len(set(call_ids)) == 2+ assert call_ids[0] == "exec_command:0"+ assert call_ids[1].startswith("call_")+ assert _tool_results(_REQUESTS[-1]) == ["did 1", "did 2"]+++def test_history_dedupe_keeps_outputs_paired_with_their_call() -> None:+ items = [+ {"type": "function_call", "call_id": "exec_command:0", "name": "a", "arguments": "{}"},+ {"type": "function_call_output", "call_id": "exec_command:0", "output": "first"},+ {"type": "function_call", "call_id": "exec_command:0", "name": "b", "arguments": "{}"},+ {"type": "function_call_output", "call_id": "exec_command:0", "output": "second"},+ ]++ rebuilt, changed = dedupe_history_call_ids(items)++ assert changed+ ids = [item["call_id"] for item in rebuilt]+ assert ids[0] == ids[1] == "exec_command:0"+ assert ids[2] == ids[3] != "exec_command:0"+ assert rebuilt[3]["output"] == "second"+++def test_history_dedupe_pairs_parallel_calls_by_order() -> None:+ items = [+ {"type": "function_call", "call_id": "dup", "name": "a", "arguments": "{}"},+ {"type": "function_call", "call_id": "dup", "name": "b", "arguments": "{}"},+ {"type": "function_call_output", "call_id": "dup", "output": "for-a"},+ {"type": "function_call_output", "call_id": "dup", "output": "for-b"},+ ]++ rebuilt, changed = dedupe_history_call_ids(items)++ assert changed+ assert rebuilt[0]["call_id"] == rebuilt[2]["call_id"] == "dup"+ assert rebuilt[1]["call_id"] == rebuilt[3]["call_id"]+ assert rebuilt[1]["call_id"] != "dup"+++def test_history_dedupe_leaves_unique_ids_alone() -> None:+ items = [+ {"type": "function_call", "call_id": "call_a", "name": "a", "arguments": "{}"},+ {"type": "function_call_output", "call_id": "call_a", "output": "x"},+ {"type": "function_call", "call_id": "call_b", "name": "b", "arguments": "{}"},+ ]++ rebuilt, changed = dedupe_history_call_ids(items)++ assert not changed+ assert rebuilt == items+++def test_turn_rewriter_is_stable_across_repeated_sightings() -> None:+ history = [{"type": "function_call", "call_id": "exec_command:0", "name": "a"}]+ rewriter = TurnCallIdRewriter(history)+ call = ResponseFunctionToolCall(+ call_id="exec_command:0", name="a", arguments="{}", type="function_call"+ )++ first = rewriter.rewrite_item(call)+ second = rewriter.rewrite_item(first)++ assert first.call_id != "exec_command:0"+ assert second.call_id == first.call_id