Agent Zero AI framework
Surface provider refusals and stop automatic retries
Alessandro89a4853310dc
diff --git a/helpers/litellm_transport.py b/helpers/litellm_transport.pyindex c65cbf63b..d9e84fce7 100644--- a/helpers/litellm_transport.py+++ b/helpers/litellm_transport.py@@ -8,6 +8,7 @@ import jsonfrom typing import Any, AsyncIterator, Iterator, Optionalfrom litellm import (+ ContentPolicyViolationError,acompletion,adelete_responses,aresponses,@@ -572,6 +573,14 @@ class ChatCompletionsTransport:chunk: Any, *, reasoning_filter: _ChatReasoningFilter | None = None) -> ChatChunk:choice = _first_choice(chunk)+ finish_reason = _get_value(choice, "finish_reason")+ if finish_reason in {"content_filter", "refusal"}:+ raise ContentPolicyViolationError(+ message=f"Model provider refused the response (finish_reason={finish_reason}). "+ "Review the request and provider restrictions before retrying.",+ model=str(_get_value(chunk, "model") or ""),+ llm_provider="",+ )delta = _get_value(choice, "delta") or {}message = _get_value(choice, "message") or _get_value(_get_value(choice, "model_extra") or {}, "message"diff --git a/helpers/litellm_transport.py.dox.md b/helpers/litellm_transport.py.dox.mdindex 27707906f..551b41259 100644--- a/helpers/litellm_transport.py.dox.md+++ b/helpers/litellm_transport.py.dox.md@@ -35,6 +35,7 @@- Fall back to Chat Completions when a Responses endpoint fails before output with an endpoint-specific server error, proxy path-unavailable error, or LiteLLM proxy-extra import error.- Fall back to Chat Completions when LiteLLM's Responses mock streaming path tries to JSON-decode a real SSE stream before any output.- Preserve Chat Completions tool calls from both non-streaming responses and streaming deltas as canonical `LLMResult` function-call items. Non-streaming calls stay canonical when accompanied by commentary; retain that commentary as a separate output message.+- Chat Completions `content_filter` and `refusal` finish reasons raise LiteLLM `ContentPolicyViolationError` on streaming and non-streaming paths; never treat blocked output as an empty success or dispatch partial tool calls. Verify with `tests/test_model_refusal.py`.- Responses messages explicitly marked `phase: commentary` produce an incremental `response_preview` with public text in `thoughts`, followed by the native tool preview. Previews reuse parsed call state; multiple calls form one parallel envelope. Only native agent-turn callbacks consume the preview: legacy callbacks, response/reasoning deltas, completed output items and call IDs remain unchanged. Unmarked/final-answer text and reasoning are not reclassified as commentary.- Preserve provider usage and LiteLLM response cost for both transports only when the response or stream actually supplies them; do not synthesize unavailable provider accounting.- Streaming Chat Completions requests include `stream_options.include_usage` so terminal usage events reach `LLMResult.usage` for OpenAI-compatible endpoints and Messages API providers alike, unless the model configuration previously rejected the option or the request already carries it. A pre-output rejection retries once without the transport-injected `include_usage` and remembers the rejection per model configuration in a process-global cache that naturally clears when the framework process restarts; tests reset it through `clear_transport_capability_cache`. User-provided `stream_options` keys are always passed through untouched, even when the provider rejects them.diff --git a/plugins/_error_retry/AGENTS.md b/plugins/_error_retry/AGENTS.mdindex ae1a15541..3fd591a45 100644--- a/plugins/_error_retry/AGENTS.md+++ b/plugins/_error_retry/AGENTS.md@@ -14,6 +14,7 @@## Local Contracts- Do not retry controlled flow exceptions such as `HandledException` and `RepairableException`.+- Do not retry LiteLLM `ContentPolicyViolationError`; leave it for the core critical-error handler to log and stop, without injecting repair instructions or clearing media.- Keep retry counts scoped per monologue.- Preserve clear agent-facing history injection when retrying a critical exception.diff --git a/plugins/_error_retry/README.md b/plugins/_error_retry/README.mdindex b2cf4caff..3e1124c76 100644--- a/plugins/_error_retry/README.md+++ b/plugins/_error_retry/README.md@@ -11,6 +11,8 @@ It does **not** retry exceptions that are already treated as controlled agent fl- `HandledException`- `RepairableException`+Provider refusals (`ContentPolicyViolationError`) also stop with a visible error instead of retrying or injecting repair instructions.+## Main Behavior- **Counter reset per monologue**diff --git a/plugins/_error_retry/extensions/python/_functions/agent/Agent/handle_exception/end/_80_retry_critical_exception.py b/plugins/_error_retry/extensions/python/_functions/agent/Agent/handle_exception/end/_80_retry_critical_exception.pyindex e920d29a7..f41be2104 100644--- a/plugins/_error_retry/extensions/python/_functions/agent/Agent/handle_exception/end/_80_retry_critical_exception.py+++ b/plugins/_error_retry/extensions/python/_functions/agent/Agent/handle_exception/end/_80_retry_critical_exception.py@@ -38,6 +38,9 @@ class RetryCriticalException(Extension):self.agent.set_data(DATA_NAME_COUNTER, 0) # reset counter if exception has been handledreturn+ if isinstance(exception, litellm.ContentPolicyViolationError):+ return # Let the critical-error handler log the refusal and stop.+cfg = plugins.get_plugin_config("_error_retry", agent=self.agent) or {}max_retries = normalize_max_retries(cfg.get("retries"))diff --git a/tests/test_error_retry_plugin.py b/tests/test_error_retry_plugin.pyindex 331c7044e..684214ebf 100644--- a/tests/test_error_retry_plugin.py+++ b/tests/test_error_retry_plugin.py@@ -120,3 +120,25 @@ def test_zero_configured_retries_disables_retry(monkeypatch):assert data["exception"] is exceptionassert agent.interventions == 0assert agent.context.log.entries == []+++def test_provider_refusal_is_left_for_critical_error_handler(monkeypatch):+ _set_retry_config(monkeypatch, 3)+ agent = FakeAgent()+ agent.agent_name = "A0"+ exception = retry_module.litellm.ContentPolicyViolationError(+ message="Provider refused", model="test", llm_provider="",+ )+ data = {"exception": exception}+ asyncio.run(retry_module.RetryCriticalException(agent=agent).execute(data))+ assert data["exception"] is exception+ assert agent.get_data(DATA_NAME_COUNTER) == 0+ assert agent.interventions == 0+ assert agent.warnings == []+ critical = importlib.import_module(+ "extensions.python._functions.agent.Agent.handle_exception.end._90_handle_critical_exception"+ )+ asyncio.run(critical.HandleCriticalException(agent=agent).execute(data))+ assert isinstance(data["exception"], retry_module.HandledException)+ assert agent.context.log.entries[0]["type"] == "error"+ assert "Provider refused" in agent.context.log.entries[0]["content"]diff --git a/tests/test_model_refusal.py b/tests/test_model_refusal.pynew file mode 100644index 000000000..d69aacabc--- /dev/null+++ b/tests/test_model_refusal.py@@ -0,0 +1,53 @@+import pytest+from litellm import ContentPolicyViolationError++import models+from helpers import litellm_transport+from helpers.litellm_transport import ChatCompletionsStreamParser, ChatCompletionsTransport++[email protected]("reason", ["content_filter", "refusal"])+def test_refusals_fail_before_partial_tool_calls_can_be_returned(reason):+ raw = {"model": "test", "choices": [{"finish_reason": reason, "message": {+ "content": "partial", "tool_calls": [{"type": "function", "function": {+ "name": "response", "arguments": '{"text":"partial"}',+ }}],+ }}]}+ for parse in (ChatCompletionsTransport.parse, ChatCompletionsStreamParser().parse):+ with pytest.raises(ContentPolicyViolationError, match=f"finish_reason={reason}"):+ parse(raw)+ assert ChatCompletionsTransport.parse({"choices": [{"finish_reason": "stop"}]}) == {+ "response_delta": "", "reasoning_delta": "",+ }++[email protected]("stream", [False, True])+async def test_refusal_stops_model_call_without_retry_and_closes_stream(monkeypatch, stream):+ calls = 0+ closed = False+ raw = {"model": "test", "choices": [{"finish_reason": "content_filter", "delta": {}}]}++ async def chunks():+ nonlocal closed+ try:+ yield raw+ finally:+ closed = True++ async def completion(**kwargs):+ nonlocal calls+ calls += 1+ return chunks() if kwargs["stream"] else raw++ async def callback(*args):+ raise AssertionError("Refused output must not reach callbacks")++ monkeypatch.setattr(litellm_transport, "acompletion", completion)+ wrapper = models.LiteLLMChatWrapper(model="test", provider="openai")+ with pytest.raises(ContentPolicyViolationError, match="Model provider refused"):+ await wrapper.unified_turn.__wrapped__(+ wrapper, messages=[], response_callback=callback if stream else None,+ )+ assert calls == 1+ assert closed == stream