🐫 CAMEL: The first and the best multi-agent framework. Finding the Scaling Law of Agents. https://www.camel-ai.org
Python agentai-societiesartificial-intelligencecommunicative-aicooperative-aideep-learninglarge-language-modelsmulti-agent-systemsnatural-language-processing
fix: try schema manual parse if `msg.parse` is `None` (#3780)
bytecii6aa602625df0
diff --git a/camel/datagen/cot_datagen.py b/camel/datagen/cot_datagen.pyindex eb72cb449..6020920c9 100644--- a/camel/datagen/cot_datagen.py+++ b/camel/datagen/cot_datagen.py@@ -20,6 +20,7 @@ from pydantic import BaseModel, Field, confloatfrom camel.agents import ChatAgentfrom camel.logger import get_logger+from camel.utils import safe_extract_parsed# Get a logger for this modulelogger = get_logger('CoTDataGenerator')@@ -200,7 +201,8 @@ class CoTDataGenerator:response = self.verifier_agent.step(prompt, response_format=VerificationResponse)- is_correct = response.msgs[0].parsed.is_correct # type:ignore [union-attr]+ parsed = safe_extract_parsed(response, VerificationResponse)+ is_correct = parsed.is_correct if parsed is not None else Falselogger.info("Answer verification result: %s", is_correct)return is_correct@@ -242,9 +244,8 @@ class CoTDataGenerator:response = self.generator_agent.step(prompt, response_format=AgentResponse)- agent_response = response.msgs[0].parsed.score # type: ignore [union-attr]-- return agent_response+ parsed = safe_extract_parsed(response, AgentResponse)+ return parsed.score if parsed is not None else 0.0def binary_search_error(self, question: str, solution: str) -> int:r"""Use binary search to locate the first error in the solution.diff --git a/camel/datagen/evol_instruct/scorer.py b/camel/datagen/evol_instruct/scorer.pyindex 1737a7184..af73001a8 100644--- a/camel/datagen/evol_instruct/scorer.py+++ b/camel/datagen/evol_instruct/scorer.py@@ -12,7 +12,6 @@# limitations under the License.# ========= Copyright 2023-2026 @ CAMEL-AI.org. All Rights Reserved. =========-import jsonfrom abc import ABC, abstractmethodfrom typing import Dict, Optional@@ -20,6 +19,7 @@ from pydantic import BaseModel, Fieldfrom camel.agents import ChatAgentfrom camel.logger import get_logger+from camel.utils import safe_extract_parsedlogger = get_logger(__name__)@@ -97,8 +97,10 @@ Respond with a JSON object like:"Try to solve the new problem. Then provide scores in JSON format.")response = self.agent.step(query, response_format=self.MathScoreSchema)- score_data = json.loads(response.msg.content)- return score_data+ parsed = safe_extract_parsed(response, self.MathScoreSchema)+ if parsed is None:+ return {"diversity": 0, "difficulty": 0, "solvability": 0}+ return parsed.model_dump()class GeneralScorer(BaseScorer):@@ -161,5 +163,7 @@ class GeneralScorer(BaseScorer):response = self.agent.step(query, response_format=self.GeneralScoreSchema)- score_data = json.loads(response.msg.content)- return score_data+ parsed = safe_extract_parsed(response, self.GeneralScoreSchema)+ if parsed is None:+ return {"diversity": 0, "complexity": 0, "validity": 0}+ return parsed.model_dump()diff --git a/camel/societies/workforce/workforce.py b/camel/societies/workforce/workforce.pyindex b758a4b44..9797e8566 100644--- a/camel/societies/workforce/workforce.py+++ b/camel/societies/workforce/workforce.py@@ -91,7 +91,7 @@ from camel.toolkits import (SearchToolkit,ThinkingToolkit,)-from camel.utils import dependencies_required+from camel.utils import dependencies_required, safe_extract_parsedfrom .events import (AllTasksCompletedEvent,@@ -1624,7 +1624,14 @@ class Workforce(BaseNode):response = self.task_agent.step(analysis_prompt, response_format=result_schema)- return response.msg.parsed+ parsed = safe_extract_parsed(response, result_schema)+ if parsed is not None:+ return parsed+ logger.warning(+ "Failed to get structured response from model, "+ "using fallback values"+ )+ return TaskAnalysisResult(**fallback_values)except Exception as e:logger.warning(diff --git a/camel/utils/__init__.py b/camel/utils/__init__.pyindex 0fc83f68a..f285f2dc3 100644--- a/camel/utils/__init__.py+++ b/camel/utils/__init__.py@@ -37,6 +37,7 @@ from .commons import (print_text_animated,retry_on_error,run_async,+ safe_extract_parsed,text_extract_from_web,to_pascal,track_agent,@@ -107,6 +108,7 @@ __all__ = ["sanitize_filename","browser_toolkit_save_auth_cookie","run_async",+ "safe_extract_parsed","configure_langfuse","is_langfuse_available","get_current_agent_session_id",diff --git a/camel/utils/commons.py b/camel/utils/commons.pyindex 37d96eba0..5b8e9ee48 100644--- a/camel/utils/commons.py+++ b/camel/utils/commons.py@@ -28,6 +28,7 @@ from functools import wrapsfrom http import HTTPStatusfrom pathlib import Pathfrom typing import (+ TYPE_CHECKING,Any,Callable,Dict,@@ -40,6 +41,9 @@ from typing import (TypeVar,cast,)++if TYPE_CHECKING:+ from camel.responses.agent_responses import ChatAgentResponsefrom urllib.parse import urlparseimport pydantic@@ -51,6 +55,7 @@ from camel.types import TaskTypefrom .constants import ConstantsF = TypeVar('F', bound=Callable[..., Any])+T = TypeVar('T', bound=BaseModel)logger = logging.getLogger(__name__)@@ -702,7 +707,7 @@ def retry_on_error(except Exception as e:last_exception = eif attempt == max_retries:- logger.error(+ logger.warning(f"Failed after {max_retries} retries: {e!s}")raise@@ -959,6 +964,65 @@ def generate_prompt_for_structured_output(return final_prompt+def safe_extract_parsed(+ response: "ChatAgentResponse",+ schema: Type[T],+) -> Optional[T]:+ r"""Safely extract a parsed structured output from a ChatAgentResponse.++ Handles the common cases where ``response.msg`` is ``None`` (empty or+ multi-message response) or ``msg.parsed`` is ``None`` (model failed to+ produce valid structured output). When the parsed value is a dict, it+ attempts to construct the schema from it.++ Args:+ response (ChatAgentResponse): The agent response to extract from.+ schema (Type[T]): The expected Pydantic model class.++ Returns:+ Optional[T]: The parsed and validated result, or ``None`` if+ extraction fails for any reason.+ """+ msg = response.msg+ # Empty or multi-message response (len(msgs) != 1)+ if msg is None:+ logger.error(+ f"safe_extract_parsed: response.msg is None "+ f"(msgs count: {len(response.msgs)}), "+ f"cannot extract {schema.__name__}"+ )+ return None+ parsed = msg.parsed+ # Already the expected Pydantic model+ if isinstance(parsed, schema):+ return parsed+ # TODO: Unify backend parsing so parsed is always a model instance,+ # removing the need for this dict fallback.+ # LiteLLM or non-OpenAI backends may return raw dict instead of model+ if isinstance(parsed, dict):+ try:+ return schema(**parsed)+ except Exception as e:+ logger.error(+ f"safe_extract_parsed: failed to construct "+ f"{schema.__name__} from dict: {e}"+ )+ return None+ # Model did not produce structured output+ if parsed is None:+ logger.error(+ f"safe_extract_parsed: msg.parsed is None, "+ f"model did not produce valid {schema.__name__}"+ )+ # Unexpected type+ else:+ logger.error(+ f"safe_extract_parsed: msg.parsed is "+ f"{type(parsed).__name__}, expected {schema.__name__}"+ )+ return None++def with_timeout(timeout=None):r"""Decorator that adds timeout functionality to functions.diff --git a/test/utils/test_commons.py b/test/utils/test_commons.pyindex 6088da9b0..3586d68df 100644--- a/test/utils/test_commons.py+++ b/test/utils/test_commons.py@@ -17,6 +17,7 @@ from unittest import TestCasefrom unittest.mock import patchimport pytest+from pydantic import BaseModelfrom camel.toolkits import FunctionToolfrom camel.toolkits.base import BaseToolkit@@ -439,3 +440,75 @@ def test_timeout_preserves_traceback():assert "deepest_function" in tb_strassert "inner_function" in tb_strassert "Error from deepest function" in str(exc_info.value)+++# --- Tests for safe_extract_parsed ---+++class _DummySchema(BaseModel):+ """Minimal Pydantic schema for safe_extract_parsed tests."""++ value: str+ score: int = 0+++def _make_response(parsed_value, num_msgs=1):+ """Build a ChatAgentResponse with a controlled parsed value."""+ from camel.messages.base import BaseMessage+ from camel.responses.agent_responses import ChatAgentResponse+ from camel.types import RoleType++ if num_msgs == 0:+ return ChatAgentResponse(msgs=[], terminated=False, info={})++ msg = BaseMessage(+ role_name="assistant",+ role_type=RoleType.ASSISTANT,+ meta_dict=None,+ content="mock",+ parsed=parsed_value,+ )+ return ChatAgentResponse(msgs=[msg], terminated=False, info={})+++def test_safe_extract_parsed_returns_valid_instance():+ from camel.utils import safe_extract_parsed++ expected = _DummySchema(value="hello", score=42)+ response = _make_response(expected)+ result = safe_extract_parsed(response, _DummySchema)+ assert result is expected+++def test_safe_extract_parsed_constructs_from_dict():+ from camel.utils import safe_extract_parsed++ response = _make_response({"value": "from_dict", "score": 7})+ result = safe_extract_parsed(response, _DummySchema)+ assert isinstance(result, _DummySchema)+ assert result.value == "from_dict"+ assert result.score == 7+++def test_safe_extract_parsed_returns_none_for_invalid_dict():+ from camel.utils import safe_extract_parsed++ response = _make_response({"bad_key": "nope"})+ result = safe_extract_parsed(response, _DummySchema)+ assert result is None+++def test_safe_extract_parsed_returns_none_when_msg_is_none():+ from camel.utils import safe_extract_parsed++ response = _make_response(None, num_msgs=0)+ result = safe_extract_parsed(response, _DummySchema)+ assert result is None+++def test_safe_extract_parsed_returns_none_when_parsed_is_none():+ from camel.utils import safe_extract_parsed++ response = _make_response(None)+ result = safe_extract_parsed(response, _DummySchema)+ assert result is Nonediff --git a/test/workforce/test_workforce.py b/test/workforce/test_workforce.pyindex a73feb570..95adb51ce 100644--- a/test/workforce/test_workforce.py+++ b/test/workforce/test_workforce.py@@ -19,8 +19,15 @@ from unittest.mock import AsyncMock, MagicMock, patchimport pytestfrom camel.agents import ChatAgent+from camel.messages.base import BaseMessagefrom camel.models import ModelFactory+from camel.responses.agent_responses import ChatAgentResponsefrom camel.societies.workforce.task_channel import TaskChannel+from camel.societies.workforce.utils import (+ FailureHandlingConfig,+ RecoveryStrategy,+ TaskAnalysisResult,+)from camel.societies.workforce.workforce import Workforcefrom camel.tasks.task import Task, TaskStatefrom camel.types import ModelPlatformType, ModelType@@ -492,3 +499,177 @@ def test_start_child_node_helper():workforce._state = WorkforceState.PAUSEDworkforce._loop = Noneworkforce._start_child_node_when_paused(mock_coroutine())+++# --- Tests for _analyze_task null-safety (use_structured_output_handler=False)+++def _make_analyze_task_workforce():+ r"""Create a Workforce with mocked internals for _analyze_task testing.++ Bypasses API-key-dependent initialization entirely by using+ object.__new__ and setting only the attributes that _analyze_task needs.+ Sets use_structured_output_handler=False to test the native structured+ output code path.+ """+ workforce = object.__new__(Workforce)+ workforce.use_structured_output_handler = False+ workforce.failure_handling_config = FailureHandlingConfig()++ mock_task_agent = MagicMock()+ mock_task_agent.reset = MagicMock()+ workforce.task_agent = mock_task_agent+ return workforce+++def _make_task():+ r"""Create a minimal Task for _analyze_task tests."""+ task = MagicMock(spec=Task)+ task.id = "test_task_1"+ task.content = "Test task content"+ task.result = "Test result"+ task.failure_count = 0+ task.get_depth.return_value = 0+ task.assigned_worker_id = "worker_1"+ return task+++def _make_mock_response(parsed_value, num_msgs=1):+ r"""Create a ChatAgentResponse with a controlled parsed value.++ Uses a real BaseMessage to satisfy Pydantic validation in+ ChatAgentResponse, with the `parsed` field set to the desired value.+ """+ from camel.types import RoleType++ if num_msgs == 0:+ return ChatAgentResponse(msgs=[], terminated=False, info={})++ msg = BaseMessage(+ role_name="assistant",+ role_type=RoleType.ASSISTANT,+ meta_dict=None,+ content="mock content",+ parsed=parsed_value,+ )+ return ChatAgentResponse(msgs=[msg], terminated=False, info={})+++def test_analyze_task_returns_valid_result():+ r"""Test that _analyze_task returns a valid TaskAnalysisResult when+ response.msg.parsed is a proper TaskAnalysisResult."""+ workforce = _make_analyze_task_workforce()+ task = _make_task()++ expected = TaskAnalysisResult(+ reasoning="Network timeout, retrying",+ recovery_strategy=RecoveryStrategy.RETRY,+ issues=["timeout"],+ )++ workforce.task_agent.step.return_value = _make_mock_response(expected)++ result = workforce._analyze_task(+ task, for_failure=True, error_message="timeout"+ )++ assert isinstance(result, TaskAnalysisResult)+ assert result.recovery_strategy == RecoveryStrategy.RETRY+ assert result.reasoning == "Network timeout, retrying"+++def test_analyze_task_fallback_when_msg_is_none():+ r"""Test that _analyze_task falls back when response.msg is None+ (e.g., empty msgs list)."""+ workforce = _make_analyze_task_workforce()+ task = _make_task()++ workforce.task_agent.step.return_value = _make_mock_response(+ None, num_msgs=0+ )++ result = workforce._analyze_task(+ task, for_failure=True, error_message="some error"+ )++ assert isinstance(result, TaskAnalysisResult)+ assert result.recovery_strategy == RecoveryStrategy.RETRY+ assert "some error" in result.issues+++def test_analyze_task_fallback_when_parsed_is_none():+ r"""Test that _analyze_task falls back when response.msg.parsed is None+ (model failed to produce structured output)."""+ workforce = _make_analyze_task_workforce()+ task = _make_task()++ workforce.task_agent.step.return_value = _make_mock_response(None)++ result = workforce._analyze_task(+ task, for_failure=True, error_message="parse failure"+ )++ assert isinstance(result, TaskAnalysisResult)+ assert result.recovery_strategy == RecoveryStrategy.RETRY+++def test_analyze_task_constructs_from_dict():+ r"""Test that _analyze_task constructs a TaskAnalysisResult when+ response.msg.parsed is a dict."""+ workforce = _make_analyze_task_workforce()+ task = _make_task()++ workforce.task_agent.step.return_value = _make_mock_response(+ {+ "reasoning": "Dict-based result",+ "recovery_strategy": RecoveryStrategy.REPLAN,+ "issues": ["incomplete"],+ }+ )++ result = workforce._analyze_task(+ task, for_failure=True, error_message="incomplete"+ )++ assert isinstance(result, TaskAnalysisResult)+ assert result.recovery_strategy == RecoveryStrategy.REPLAN+ assert result.reasoning == "Dict-based result"+++def test_analyze_task_fallback_when_dict_is_invalid():+ r"""Test that _analyze_task falls back when response.msg.parsed is a dict+ with invalid keys that fail schema validation."""+ workforce = _make_analyze_task_workforce()+ task = _make_task()++ workforce.task_agent.step.return_value = _make_mock_response(+ {+ "bad_key": "unexpected value",+ "another_bad_key": 123,+ }+ )++ result = workforce._analyze_task(+ task, for_failure=True, error_message="bad dict"+ )++ assert isinstance(result, TaskAnalysisResult)+ assert result.recovery_strategy == RecoveryStrategy.RETRY+ assert "bad dict" in result.issues+++def test_analyze_task_quality_eval_fallback():+ r"""Test that _analyze_task returns quality evaluation fallback when+ response.msg is None and for_failure=False."""+ workforce = _make_analyze_task_workforce()+ task = _make_task()++ workforce.task_agent.step.return_value = _make_mock_response(+ None, num_msgs=0+ )++ result = workforce._analyze_task(task, for_failure=False)++ assert isinstance(result, TaskAnalysisResult)+ assert result.quality_score == 80+ assert result.recovery_strategy is None