Skip to content

armgit / strix

Public Imported from GitHub
0

Open-source AI penetration testing tool to find and fix your app’s vulnerabilities.

fix(core): recover from hallucinated tool names instead of ending the scan

Ahmed Allam4a455b1e620e
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index f0a96b39..7ceb3cc4 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -21,7 +21,7 @@ repos:
fastapi,
pytest,
hatchling,
- "openai-agents[litellm]==0.14.6",
+ "openai-agents[litellm]>=0.19.0,<0.20",
]
args: [--install-types, --non-interactive]
diff --git a/pyproject.toml b/pyproject.toml
index 0655cf68..6a2ac4b0 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -33,8 +33,8 @@ classifiers = [
"Programming Language :: Python :: 3.14",
]
dependencies = [
- "openai-agents[litellm]==0.14.6",
- "openai>=2.26.0,<2.45",
+ "openai-agents[litellm]>=0.19.0,<0.20",
+ "openai>=2.45.0,<3",
"litellm",
"pydantic>=2.11.3",
"pydantic-settings>=2.13.0",
@@ -233,6 +233,7 @@ ignore = [
"strix/interface/auth_cli.py" = ["N802"]
"tests/test_codex_streaming.py" = ["N802"]
"tests/test_disable_streaming.py" = ["N802"]
+"tests/test_unknown_tool_recovery.py" = ["N802"]
"tests/test_report_pdf.py" = ["S105", "S106"]
# Stdlib HTTP handler overrides (do_GET/do_POST) and lazy imports that avoid a
# circular dependency with strix.telemetry / strix.interface.viewer.report_pdf.
diff --git a/strix/core/runner.py b/strix/core/runner.py
index 79e9fff2..8d75939f 100644
--- a/strix/core/runner.py
+++ b/strix/core/runner.py
@@ -268,6 +268,9 @@ async def run_strix_scan(
model_settings=model_settings,
sandbox=SandboxRunConfig(client=bundle["client"], session=bundle["session"]),
trace_include_sensitive_data=False,
+ # A hallucinated tool name is a recoverable model mistake, not a scan-ending
+ # error: hand it back as a tool result so the agent can correct itself.
+ tool_not_found_behavior="return_error_to_model",
)
hooks = ReportUsageHooks(
model=resolved_model,
diff --git a/tests/test_runner_rate_limit.py b/tests/test_runner_rate_limit.py
index 061ad3c5..3110ae2c 100644
--- a/tests/test_runner_rate_limit.py
+++ b/tests/test_runner_rate_limit.py
@@ -8,6 +8,7 @@ from typing import Any
import httpx
import pytest
+from agents import ModelSettings
from openai import RateLimitError
import strix.tools.notes.tools as notes_tools
@@ -64,7 +65,7 @@ async def test_persistent_rate_limit_stops_gracefully(
monkeypatch.setattr(runner, "build_root_task", lambda _scan_config: "task")
monkeypatch.setattr(runner, "build_scope_context", lambda _scan_config: "")
- monkeypatch.setattr(runner, "make_model_settings", lambda *_args, **_kwargs: object())
+ monkeypatch.setattr(runner, "make_model_settings", lambda *_args, **_kwargs: ModelSettings())
monkeypatch.setattr(runner, "build_strix_agent", lambda **_kwargs: object())
monkeypatch.setattr(runner, "make_child_factory", lambda **_kwargs: lambda **_k: object())
monkeypatch.setattr(runner, "open_agent_session", lambda _root_id, _db: object())
diff --git a/tests/test_runner_root_prompt.py b/tests/test_runner_root_prompt.py
index cd4d4ac8..2c346203 100644
--- a/tests/test_runner_root_prompt.py
+++ b/tests/test_runner_root_prompt.py
@@ -11,6 +11,7 @@ from typing import Any
import httpx
import pytest
+from agents import ModelSettings
from openai import RateLimitError
import strix.tools.notes.tools as notes_tools
@@ -74,7 +75,7 @@ def _patch_engine_scaffold(
monkeypatch.setattr(runner, "build_root_task", lambda _scan_config: "task")
monkeypatch.setattr(runner, "build_scope_context", lambda _scan_config: scope_context)
- monkeypatch.setattr(runner, "make_model_settings", lambda *_args, **_kwargs: object())
+ monkeypatch.setattr(runner, "make_model_settings", lambda *_args, **_kwargs: ModelSettings())
captured: dict[str, Any] = {}
@@ -87,7 +88,8 @@ def _patch_engine_scaffold(
monkeypatch.setattr(runner, "make_child_factory", lambda **_kwargs: lambda **_k: object())
monkeypatch.setattr(runner, "open_agent_session", lambda _root_id, _db: object())
- async def _raise_rate_limit(*_args: Any, **_kwargs: Any) -> None:
+ async def _raise_rate_limit(*_args: Any, **kwargs: Any) -> None:
+ captured["run_config"] = kwargs.get("run_config")
raise _make_rate_limit_error()
monkeypatch.setattr(runner, "run_agent_loop", _raise_rate_limit)
@@ -176,3 +178,21 @@ async def test_root_prompt_options_default_to_none(
kwargs = captured["kwargs"]
assert kwargs["instructions_override"] is None
assert kwargs["system_prompt_context"] == {"scope": "built-in"}
+
+
+async def test_unknown_tool_calls_are_returned_to_the_model(
+ monkeypatch: pytest.MonkeyPatch,
+ tmp_path: Any,
+) -> None:
+ """A hallucinated tool name must not end the scan."""
+ captured = _patch_engine_scaffold(monkeypatch, tmp_path, {})
+
+ await runner.run_strix_scan(
+ scan_config={"targets": [], "scan_mode": "deep"},
+ scan_id="scan-unknown-tool",
+ image="img",
+ coordinator=AgentCoordinator(),
+ )
+
+ assert captured["run_config"].tool_not_found_behavior == "return_error_to_model"
diff --git a/tests/test_unknown_tool_recovery.py b/tests/test_unknown_tool_recovery.py
new file mode 100644
index 00000000..bc6e747e
--- /dev/null
+++ b/tests/test_unknown_tool_recovery.py
@@ -0,0 +1,155 @@
+"""Tests for surviving a hallucinated tool name.
+
+Models regularly invent tool names that Strix does not register (``read_file``
+is a common one, borrowed from other agent frameworks). The SDK's default is to
+raise ``ModelBehaviorError``, which ends the whole run: nothing in Strix retries
+it, so one bad token discards a scan. The runner therefore opts into
+``tool_not_found_behavior="return_error_to_model"`` so the unknown call comes
+back as a tool result and the agent corrects itself on the next turn.
+"""
+
+from __future__ import annotations
+
+import json
+import threading
+from http.server import BaseHTTPRequestHandler, HTTPServer
+from typing import TYPE_CHECKING, Any
+
+import pytest
+from agents import Agent, Runner, function_tool
+from agents.exceptions import ModelBehaviorError
+from agents.models.interface import Model, ModelProvider
+from agents.models.openai_chatcompletions import OpenAIChatCompletionsModel
+from agents.run import RunConfig
+from openai import AsyncOpenAI
+
+from strix.config.models import _NonStreamingModel
+
+
+if TYPE_CHECKING:
+ from collections.abc import Iterator
+
+
+_TURNS: list[dict[str, Any]] = []
+
+
+def _unknown_tool_call_completion() -> dict[str, Any]:
+ return {
+ "id": "chatcmpl-1",
+ "object": "chat.completion",
+ "created": 0,
+ "model": "gw-model",
+ "choices": [
+ {
+ "index": 0,
+ "finish_reason": "tool_calls",
+ "message": {
+ "role": "assistant",
+ "content": None,
+ "tool_calls": [
+ {
+ "id": "call_1",
+ "type": "function",
+ "function": {
+ "name": "read_file",
+ "arguments": '{"path": "/etc/passwd"}',
+ },
+ }
+ ],
+ },
+ }
+ ],
+ "usage": {"prompt_tokens": 5, "completion_tokens": 2, "total_tokens": 7},
+ }
+
+
+def _text_completion(text: str) -> dict[str, Any]:
+ return {
+ "id": "chatcmpl-2",
+ "object": "chat.completion",
+ "created": 0,
+ "model": "gw-model",
+ "choices": [
+ {"index": 0, "finish_reason": "stop", "message": {"role": "assistant", "content": text}}
+ ],
+ "usage": {"prompt_tokens": 5, "completion_tokens": 3, "total_tokens": 8},
+ }
+
+
+class _Handler(BaseHTTPRequestHandler):
+ """Calls an unregistered tool on turn 1, then answers on turn 2."""
+
+ def log_message(self, *args: Any) -> None:
+ pass
+
+ def do_POST(self) -> None:
+ length = int(self.headers.get("Content-Length", 0))
+ _TURNS.append(json.loads(self.rfile.read(length) or b"{}"))
+ completion = (
+ _unknown_tool_call_completion() if len(_TURNS) == 1 else _text_completion("recovered")
+ )
+ payload = json.dumps(completion).encode()
+ self.send_response(200)
+ self.send_header("Content-Type", "application/json")
+ self.send_header("Content-Length", str(len(payload)))
+ self.end_headers()
+ self.wfile.write(payload)
+
+
+def gateway_url() -> Iterator[str]:
+ _TURNS.clear()
+ server = HTTPServer(("127.0.0.1", 0), _Handler)
+ thread = threading.Thread(target=server.serve_forever, daemon=True)
+ thread.start()
+ try:
+ yield f"http://127.0.0.1:{server.server_address[1]}/v1"
+ finally:
+ server.shutdown()
+ server.server_close()
+
+
+def _agent() -> Agent[Any]:
+ @function_tool
+ def real_tool(n: int) -> str:
+ return f"did {n}"
+
+ return Agent(name="Strix", instructions="test", tools=[real_tool], model="gw-model")
+
+
+def _run_config(base_url: str, **kwargs: Any) -> RunConfig:
+ class _Provider(ModelProvider):
+ def get_model(self, model_name: str | None) -> Model: # noqa: ARG002
+ client = AsyncOpenAI(api_key="tok", base_url=base_url)
+ return _NonStreamingModel(OpenAIChatCompletionsModel("gw-model", openai_client=client))
+
+ return RunConfig(model_provider=_Provider(), **kwargs)
+
+
+async def test_unknown_tool_call_is_returned_to_the_model(gateway_url: str) -> None:
+ result = Runner.run_streamed(
+ _agent(),
+ input="go",
+ run_config=_run_config(gateway_url, tool_not_found_behavior="return_error_to_model"),
+ )
+ async for _ in result.stream_events():
+ pass
+
+ assert result.final_output == "recovered"
+ # The second turn carries the error back to the model as a tool result.
+ tool_results = [
+ item
+ for item in _TURNS[1]["messages"]
+ if item.get("role") == "tool" and item.get("tool_call_id") == "call_1"
+ ]
+ assert tool_results
+ assert "read_file" in str(tool_results[0]["content"])
+
+
+async def test_unknown_tool_call_kills_the_run_without_the_setting(gateway_url: str) -> None:
+ result = Runner.run_streamed(_agent(), input="go", run_config=_run_config(gateway_url))
+ with pytest.raises(ModelBehaviorError, match="read_file"):
+ async for _ in result.stream_events():
+ pass
diff --git a/uv.lock b/uv.lock
index 8b933b4a..b98bac5c 100644
--- a/uv.lock
+++ b/uv.lock
@@ -1384,7 +1384,7 @@ wheels = [
[[package]]
name = "openai"
-version = "2.44.0"
+version = "2.53.0"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "anyio" },
@@ -1396,14 +1396,14 @@ dependencies = [
{ name = "tqdm" },
{ name = "typing-extensions" },
]
-sdist = { url = "https://files.pythonhosted.org/packages/49/f5/7c7cb955305cb41f7f3c5fd7e0e38bf6bbf2658468863d4b7b868a5cb8df/openai-2.44.0.tar.gz", hash = "sha256:68a5a5ffad82b8ff7d451c437529fb64f7c3b8123aaf0c021966a882d9e3947d", size = 988753, upload-time = "2026-06-24T20:56:02.293Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/ef/cf/36e3e7235fdf6d125c052acc0970924611b17a20a4fe580596faf4566a65/openai-2.53.0.tar.gz", hash = "sha256:baf5802ad08980e1d9d561e1b996e800c8bcd14af5847c6d0e7a5cc59e4d4116", size = 1099435, upload-time = "2026-08-03T21:42:01.664Z" }
wheels = [
- { url = "https://files.pythonhosted.org/packages/ae/f4/561ed79fd94876160018a5e75254cfcb9b0e62d4dded9dcb20072e86d623/openai-2.44.0-py3-none-any.whl", hash = "sha256:0a2a3ab2e29aeda368700f662ff9ba0f9df17ba4c54577a64e08b8115a3cc0ad", size = 1366216, upload-time = "2026-06-24T20:55:58.882Z" },
+ { url = "https://files.pythonhosted.org/packages/78/0f/cc6afea3542a5142c5d8fc8211c5e059a8375105d004a41dfa2c7948dbb0/openai-2.53.0-py3-none-any.whl", hash = "sha256:c694ffc747a3c4d1663ef2b07b811315a476164ee5efa3a993967349ebca7618", size = 1659829, upload-time = "2026-08-03T21:41:59.581Z" },
]
[[package]]
name = "openai-agents"
-version = "0.14.6"
+version = "0.19.0"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "griffelib" },
@@ -1411,13 +1411,12 @@ dependencies = [
{ name = "openai" },
{ name = "pydantic" },
{ name = "requests" },
- { name = "types-requests" },
{ name = "typing-extensions" },
{ name = "websockets" },
]
-sdist = { url = "https://files.pythonhosted.org/packages/d5/fe/4f859d13ba5eea5fe5a3166ffeed04bd04d478ccf3187da6acebb17ba2a7/openai_agents-0.14.6.tar.gz", hash = "sha256:e9d16b835f73be4c5e3798694f90d7a62efcade931e59416bc7462c850e15705", size = 5311175, upload-time = "2026-04-25T02:32:00.897Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/ba/6c/8fa83cb23d2fe864b284cb45acf895d72a2f6e9827cc684dd4ef0d02d414/openai_agents-0.19.0.tar.gz", hash = "sha256:1d519d6966834e5c04160caec3a2549190e92ce50cdeb22fac5e17e67b8b98b2", size = 5620718, upload-time = "2026-07-27T22:49:26.615Z" }
wheels = [
- { url = "https://files.pythonhosted.org/packages/4b/96/b49d04e860c79699814289c273e88066ce97a50686172b5733b7458da062/openai_agents-0.14.6-py3-none-any.whl", hash = "sha256:fdd3fb459892c8af5d0b522908b544e96f6217c7254ba55e966424493b43c1ed", size = 816112, upload-time = "2026-04-25T02:31:58.976Z" },
+ { url = "https://files.pythonhosted.org/packages/3b/3a/bac1aa3405c0f11b4334ac999e881d08fa40fad1f3b7229a1ca222ada489/openai_agents-0.19.0-py3-none-any.whl", hash = "sha256:25392cff993eca7c75b0679ec8d0111faef20c517222c0193111097d0f70db7a", size = 928463, upload-time = "2026-07-27T22:49:24.405Z" },
]
[package.optional-dependencies]
@@ -2426,8 +2425,8 @@ requires-dist = [
{ name = "docker", specifier = ">=7.1.0" },
{ name = "google-auth", marker = "extra == 'vertex'", specifier = ">=2.0.0" },
{ name = "litellm" },
- { name = "openai", specifier = ">=2.26.0,<2.45" },
- { name = "openai-agents", extras = ["litellm"], specifier = "==0.14.6" },
+ { name = "openai", specifier = ">=2.45.0,<3" },
+ { name = "openai-agents", extras = ["litellm"], specifier = ">=0.19.0,<0.20" },
{ name = "pydantic", specifier = ">=2.11.3" },
{ name = "pydantic-settings", specifier = ">=2.13.0" },
{ name = "pypdf", specifier = ">=5.0" },
@@ -2550,18 +2549,6 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/3f/f9/2b3ff4e56e5fa7debfaf9eb135d0da96f3e9a1d5b27222223c7296336e5f/typer-0.25.1-py3-none-any.whl", hash = "sha256:75caa44ed46a03fb2dab8808753ffacdbfea88495e74c85a28c5eefcf5f39c89", size = 58409, upload-time = "2026-04-30T19:32:18.271Z" },
]
-[[package]]
-name = "types-requests"
-version = "2.33.0.20260518"
-source = { registry = "https://pypi.org/simple" }
-dependencies = [
- { name = "urllib3" },
-]
-sdist = { url = "https://files.pythonhosted.org/packages/e0/01/c5a19253fe1ac159159ddf9a3a07cec8bb5e486ec4d9002ad2821da0e5d2/types_requests-2.33.0.20260518.tar.gz", hash = "sha256:df7bd3bfe0ca8402dfb841e7d9be714bb5578203283d66d7dc4ef69343449a5e", size = 24752, upload-time = "2026-05-18T06:07:37.966Z" }
-wheels = [
- { url = "https://files.pythonhosted.org/packages/1c/bc/b139710a3b6018f7fb2b9508b35c8af564e61bf2bf4fa619d088f3e16f85/types_requests-2.33.0.20260518-py3-none-any.whl", hash = "sha256:626d697d1adaaff76e2044dc8c5c051d8f21abc157bdfe204a75558076fe0bf0", size = 21391, upload-time = "2026-05-18T06:07:37.044Z" },
-]
-
[[package]]
name = "typing-extensions"
version = "4.15.0"