Skip to content

AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters.

fix(blocks): raise AIConditionBlock errors instead of swallowing them (#12593)

Zamil Majdy68f5d2ad08e7
diff --git a/autogpt_platform/backend/backend/blocks/ai_condition.py b/autogpt_platform/backend/backend/blocks/ai_condition.py
index c28c1e9f7d..6d62d4ab77 100644
--- a/autogpt_platform/backend/backend/blocks/ai_condition.py
+++ b/autogpt_platform/backend/backend/blocks/ai_condition.py
@@ -1,3 +1,4 @@
+import re
from typing import Any
from backend.blocks._base import (
@@ -19,6 +20,33 @@ from backend.blocks.llm import (
)
from backend.data.model import APIKeyCredentials, NodeExecutionStats, SchemaField
+# Minimum max_output_tokens accepted by OpenAI-compatible APIs.
+# A true/false answer fits comfortably within this budget.
+MIN_LLM_OUTPUT_TOKENS = 16
+
+
+def _parse_boolean_response(response_text: str) -> tuple[bool, str | None]:
+ """Parse an LLM response into a boolean result.
+
+ Returns a ``(result, error)`` tuple. *error* is ``None`` when the
+ response is unambiguous; otherwise it contains a diagnostic message
+ and *result* defaults to ``False``.
+ """
+ text = response_text.strip().lower()
+ if text == "true":
+ return True, None
+ if text == "false":
+ return False, None
+
+ # Fuzzy match – use word boundaries to avoid false positives like "untrue".
+ tokens = set(re.findall(r"\b(true|false|yes|no|1|0)\b", text))
+ if tokens == {"true"} or tokens == {"yes"} or tokens == {"1"}:
+ return True, None
+ if tokens == {"false"} or tokens == {"no"} or tokens == {"0"}:
+ return False, None
+
+ return False, f"Unclear AI response: '{response_text}'"
+
class AIConditionBlock(AIBlockBase):
"""
@@ -162,54 +190,26 @@ class AIConditionBlock(AIBlockBase):
]
# Call the LLM
- try:
- response = await self.llm_call(
- credentials=credentials,
- llm_model=input_data.model,
- prompt=prompt,
- max_tokens=10, # We only expect a true/false response
- )
-
- # Extract the boolean result from the response
- response_text = response.response.strip().lower()
- if response_text == "true":
- result = True
- elif response_text == "false":
- result = False
- else:
- # If the response is not clear, try to interpret it using word boundaries
- import re
-
- # Use word boundaries to avoid false positives like 'untrue' or '10'
- tokens = set(re.findall(r"\b(true|false|yes|no|1|0)\b", response_text))
-
- if tokens == {"true"} or tokens == {"yes"} or tokens == {"1"}:
- result = True
- elif tokens == {"false"} or tokens == {"no"} or tokens == {"0"}:
- result = False
- else:
- # Unclear or conflicting response - default to False and yield error
- result = False
- yield "error", f"Unclear AI response: '{response.response}'"
-
- # Update internal stats
- self.merge_stats(
- NodeExecutionStats(
- input_token_count=response.prompt_tokens,
- output_token_count=response.completion_tokens,
- )
- )
- self.prompt = response.prompt
+ response = await self.llm_call(
+ credentials=credentials,
+ llm_model=input_data.model,
+ prompt=prompt,
+ max_tokens=MIN_LLM_OUTPUT_TOKENS,
+ )
- except Exception as e:
- # In case of any error, default to False to be safe
- result = False
- # Log the error but don't fail the block execution
- import logging
+ # Extract the boolean result from the response
+ result, error = _parse_boolean_response(response.response)
+ if error:
+ yield "error", error
- logger = logging.getLogger(__name__)
- logger.error(f"AI condition evaluation failed: {str(e)}")
- yield "error", f"AI evaluation failed: {str(e)}"
+ # Update internal stats
+ self.merge_stats(
+ NodeExecutionStats(
+ input_token_count=response.prompt_tokens,
+ output_token_count=response.completion_tokens,
+ )
+ )
+ self.prompt = response.prompt
# Yield results
yield "result", result
diff --git a/autogpt_platform/backend/backend/blocks/ai_condition_test.py b/autogpt_platform/backend/backend/blocks/ai_condition_test.py
new file mode 100644
index 0000000000..babb1eb4cf
--- /dev/null
+++ b/autogpt_platform/backend/backend/blocks/ai_condition_test.py
@@ -0,0 +1,147 @@
+"""Tests for AIConditionBlock – regression coverage for max_tokens and error propagation."""
+
+from __future__ import annotations
+
+from typing import cast
+
+import pytest
+
+from backend.blocks.ai_condition import (
+ MIN_LLM_OUTPUT_TOKENS,
+ AIConditionBlock,
+ _parse_boolean_response,
+)
+from backend.blocks.llm import (
+ DEFAULT_LLM_MODEL,
+ TEST_CREDENTIALS,
+ TEST_CREDENTIALS_INPUT,
+ AICredentials,
+ LLMResponse,
+)
+
+_TEST_AI_CREDENTIALS = cast(AICredentials, TEST_CREDENTIALS_INPUT)
+
+
+# ---------------------------------------------------------------------------
+# Helper to collect all yields from the async generator
+# ---------------------------------------------------------------------------
+
+
+async def _collect_outputs(block: AIConditionBlock, input_data, credentials):
+ outputs: dict[str, object] = {}
+ async for name, value in block.run(input_data, credentials=credentials):
+ outputs[name] = value
+ return outputs
+
+
+def _make_input(**overrides) -> AIConditionBlock.Input:
+ defaults: dict = {
+ "input_value": "[email protected]",
+ "condition": "the input is an email address",
+ "yes_value": "yes!",
+ "no_value": "no!",
+ "model": DEFAULT_LLM_MODEL,
+ "credentials": TEST_CREDENTIALS_INPUT,
+ }
+ defaults.update(overrides)
+ return AIConditionBlock.Input(**defaults)
+
+
+def _mock_llm_response(response_text: str) -> LLMResponse:
+ return LLMResponse(
+ raw_response="",
+ prompt=[],
+ response=response_text,
+ tool_calls=None,
+ prompt_tokens=10,
+ completion_tokens=5,
+ reasoning=None,
+ )
+
+
+# ---------------------------------------------------------------------------
+# _parse_boolean_response unit tests
+# ---------------------------------------------------------------------------
+
+
+class TestParseBooleanResponse:
+ def test_true_exact(self):
+ assert _parse_boolean_response("true") == (True, None)
+
+ def test_false_exact(self):
+ assert _parse_boolean_response("false") == (False, None)
+
+ def test_true_with_whitespace(self):
+ assert _parse_boolean_response(" True ") == (True, None)
+
+ def test_yes_fuzzy(self):
+ assert _parse_boolean_response("Yes") == (True, None)
+
+ def test_no_fuzzy(self):
+ assert _parse_boolean_response("no") == (False, None)
+
+ def test_one_fuzzy(self):
+ assert _parse_boolean_response("1") == (True, None)
+
+ def test_zero_fuzzy(self):
+ assert _parse_boolean_response("0") == (False, None)
+
+ def test_unclear_response(self):
+ result, error = _parse_boolean_response("I'm not sure")
+ assert result is False
+ assert error is not None
+ assert "Unclear" in error
+
+ def test_conflicting_tokens(self):
+ result, error = _parse_boolean_response("true and false")
+ assert result is False
+ assert error is not None
+
+
+# ---------------------------------------------------------------------------
+# Regression: max_tokens is set to MIN_LLM_OUTPUT_TOKENS
+# ---------------------------------------------------------------------------
+
+
+class TestMaxTokensRegression:
+ @pytest.mark.asyncio
+ async def test_llm_call_receives_min_output_tokens(self):
+ """max_tokens must be MIN_LLM_OUTPUT_TOKENS (16) – the previous value
+ of 1 was too low and caused OpenAI to reject the request."""
+ block = AIConditionBlock()
+ captured_kwargs: dict = {}
+
+ async def spy_llm_call(**kwargs):
+ captured_kwargs.update(kwargs)
+ return _mock_llm_response("true")
+
+ block.llm_call = spy_llm_call # type: ignore[assignment]
+
+ input_data = _make_input()
+ await _collect_outputs(block, input_data, credentials=TEST_CREDENTIALS)
+
+ assert captured_kwargs["max_tokens"] == MIN_LLM_OUTPUT_TOKENS
+ assert captured_kwargs["max_tokens"] == 16
+
+
+# ---------------------------------------------------------------------------
+# Regression: exceptions from llm_call must propagate
+# ---------------------------------------------------------------------------
+
+
+class TestExceptionPropagation:
+ @pytest.mark.asyncio
+ async def test_llm_call_exception_propagates(self):
+ """If llm_call raises, the exception must NOT be swallowed.
+ Previously the block caught all exceptions and silently returned
+ result=False."""
+ block = AIConditionBlock()
+
+ async def boom(**kwargs):
+ raise RuntimeError("LLM provider error")
+
+ block.llm_call = boom # type: ignore[assignment]
+
+ input_data = _make_input()
+ with pytest.raises(RuntimeError, match="LLM provider error"):
+ await _collect_outputs(block, input_data, credentials=TEST_CREDENTIALS)