Open-source AI penetration testing tool to find and fix your app’s vulnerabilities.
Python agentsai-hackingai-penetration-testingai-pentestingai-securityartificial-intelligencebug-bountycode-qualityctf-toolscybersecuritycybersecurity-toolsethical-hackinghackingllm-securityoffensive-securitypenetration-testingpentesting-toolsred-teamingsecuritysecurity-automation
fix(llm): surface structured provider refusals (#944)
alex sa9deb84260c3
diff --git a/strix/core/execution.py b/strix/core/execution.pyindex ae726ed3..7272a8fe 100644--- a/strix/core/execution.py+++ b/strix/core/execution.py@@ -55,6 +55,23 @@ _INPUT_REJECTION_CODES = frozenset({400, 404, 422})_MAX_COMPACTIONS_PER_CYCLE = 2+class ProviderRefusalError(AgentsException):+ """Raised when a provider returns a structured refusal instead of an exception."""+++def _structured_provider_refusal(result: Any) -> str | None:+ for item in getattr(result, "new_items", ()) or ():+ raw_item = getattr(item, "raw_item", None)+ for content in getattr(raw_item, "content", ()) or ():+ if getattr(content, "type", None) != "refusal":+ continue+ refusal = getattr(content, "refusal", None)+ if isinstance(refusal, str) and refusal.strip():+ return refusal.strip()+ return "The model provider refused this request."+ return None++def _run_config_model(run_config: RunConfig) -> str | None:return run_config.model if isinstance(run_config.model, str) else None@@ -490,6 +507,8 @@ async def _run_cycle( # noqa: PLR0912, PLR0915logger.exception("stream event sink failed for %s", agent_id)if stream.run_loop_exception is not None:raise stream.run_loop_exception+ if refusal := _structured_provider_refusal(stream):+ raise ProviderRefusalError(refusal)except (BudgetExceededError, BudgetPausedError, SubagentBudgetReservedError):raiseexcept RuntimeError as stream_exc:@@ -584,6 +603,11 @@ async def _run_cycle( # noqa: PLR0912, PLR0915return await _handle_content_guardrail(coordinator, agent_id, exc, interactive=interactive)+ if isinstance(exc, ProviderRefusalError):+ logger.warning("agent %s refused by the model provider: %s", agent_id, exc)+ await coordinator.set_status(agent_id, "failed", error=str(exc))+ await _notify_parent_on_terminal(coordinator, agent_id, "failed")+ return Noneif not interactive:raiseif isinstance(exc, MaxTurnsExceeded):diff --git a/tests/test_execution.py b/tests/test_execution.pyindex 2ecd9a84..8fa043f1 100644--- a/tests/test_execution.py+++ b/tests/test_execution.py@@ -9,8 +9,10 @@ from typing import Anyfrom unittest.mock import MagicMockimport pytest+from agents.items import MessageOutputItemfrom agents.memory import SQLiteSessionfrom agents.tool_context import ToolContext+from openai.types.responses import ResponseOutputMessage, ResponseOutputRefusalfrom strix.config import codexfrom strix.core import execution@@ -24,6 +26,30 @@ from strix.core.execution import (from strix.tools.finish.tool import finish_scan+class _StructuredRefusalStream:+ def __init__(self, refusal: str) -> None:+ self.run_loop_exception: BaseException | None = None+ self.new_items = [+ MessageOutputItem(+ agent=MagicMock(),+ raw_item=ResponseOutputMessage(+ id="msg-refusal",+ content=[ResponseOutputRefusal(type="refusal", refusal=refusal)],+ role="assistant",+ status="completed",+ type="message",+ ),+ )+ ]++ async def stream_events(self) -> Any:+ if False:+ yield None++ def cancel(self, mode: str = "immediate") -> None: # noqa: ARG002+ return++async def _call_finish_scan(coordinator: AgentCoordinator, agent_id: str, parent_id: str | None) -> dict[str, Any]:@@ -544,6 +570,70 @@ async def test_guardrail_noninteractive_fails_only_blocked_agent(tmp_path: Any)session.close()+async def test_structured_provider_refusal_fails_interactive_agent(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ refusal = "This request was blocked under the provider's usage policy."+ stream = _StructuredRefusalStream(refusal)+ monkeypatch.setattr(execution.Runner, "run_streamed", lambda *_args, **_kwargs: stream)+ coordinator = AgentCoordinator()+ await coordinator.register("root", "strix", parent_id=None)++ result = await execution._run_cycle(+ MagicMock(),+ coordinator,+ "root",+ input_data="task",+ run_config=MagicMock(),+ context={},+ max_turns=5,+ session=None,+ interactive=True,+ event_sink=None,+ hooks=None,+ )++ assert result is None+ assert coordinator.statuses["root"] == "failed"+ assert coordinator.errors["root"] == refusal+++async def test_structured_provider_refusal_fails_noninteractive_child(+ tmp_path: Any,+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ refusal = "This request was blocked under the provider's usage policy."+ stream = _StructuredRefusalStream(refusal)+ monkeypatch.setattr(execution.Runner, "run_streamed", lambda *_args, **_kwargs: stream)+ coordinator = AgentCoordinator()+ await coordinator.register("root", "strix", parent_id=None)+ await coordinator.register("child", "recon", parent_id="root")+ session = SQLiteSession("root", tmp_path / "agents.db")+ await coordinator.attach_runtime("root", session=session)++ result = await execution._run_cycle(+ MagicMock(),+ coordinator,+ "child",+ input_data="task",+ run_config=MagicMock(),+ context={"parent_id": "root"},+ max_turns=5,+ session=None,+ interactive=False,+ event_sink=None,+ hooks=None,+ )++ assert result is None+ assert coordinator.statuses["child"] == "failed"+ assert coordinator.errors["child"] == refusal+ assert coordinator.pending_counts.get("root", 0) > 0+ session.close()++@pytest.mark.asyncioasync def test_resume_revives_guardrail_parked_child_but_not_plain_waiting(tmp_path: Any, monkeypatch: pytest.MonkeyPatch