Skip to content

armgit / camel

Public Imported from GitHub
0

🐫 CAMEL: The first and the best multi-agent framework. Finding the Scaling Law of Agents. https://www.camel-ai.org

fix: try schema manual parse if `msg.parse` is `None` (#3780)

bytecii6aa602625df0
diff --git a/camel/datagen/cot_datagen.py b/camel/datagen/cot_datagen.py
index eb72cb449..6020920c9 100644
--- a/camel/datagen/cot_datagen.py
+++ b/camel/datagen/cot_datagen.py
@@ -20,6 +20,7 @@ from pydantic import BaseModel, Field, confloat
from camel.agents import ChatAgent
from camel.logger import get_logger
+from camel.utils import safe_extract_parsed
# Get a logger for this module
logger = get_logger('CoTDataGenerator')
@@ -200,7 +201,8 @@ class CoTDataGenerator:
response = self.verifier_agent.step(
prompt, response_format=VerificationResponse
)
- is_correct = response.msgs[0].parsed.is_correct # type:ignore [union-attr]
+ parsed = safe_extract_parsed(response, VerificationResponse)
+ is_correct = parsed.is_correct if parsed is not None else False
logger.info("Answer verification result: %s", is_correct)
return is_correct
@@ -242,9 +244,8 @@ class CoTDataGenerator:
response = self.generator_agent.step(
prompt, response_format=AgentResponse
)
- agent_response = response.msgs[0].parsed.score # type: ignore [union-attr]
-
- return agent_response
+ parsed = safe_extract_parsed(response, AgentResponse)
+ return parsed.score if parsed is not None else 0.0
def binary_search_error(self, question: str, solution: str) -> int:
r"""Use binary search to locate the first error in the solution.
diff --git a/camel/datagen/evol_instruct/scorer.py b/camel/datagen/evol_instruct/scorer.py
index 1737a7184..af73001a8 100644
--- a/camel/datagen/evol_instruct/scorer.py
+++ b/camel/datagen/evol_instruct/scorer.py
@@ -12,7 +12,6 @@
# limitations under the License.
# ========= Copyright 2023-2026 @ CAMEL-AI.org. All Rights Reserved. =========
-import json
from abc import ABC, abstractmethod
from typing import Dict, Optional
@@ -20,6 +19,7 @@ from pydantic import BaseModel, Field
from camel.agents import ChatAgent
from camel.logger import get_logger
+from camel.utils import safe_extract_parsed
logger = get_logger(__name__)
@@ -97,8 +97,10 @@ Respond with a JSON object like:
"Try to solve the new problem. Then provide scores in JSON format."
)
response = self.agent.step(query, response_format=self.MathScoreSchema)
- score_data = json.loads(response.msg.content)
- return score_data
+ parsed = safe_extract_parsed(response, self.MathScoreSchema)
+ if parsed is None:
+ return {"diversity": 0, "difficulty": 0, "solvability": 0}
+ return parsed.model_dump()
class GeneralScorer(BaseScorer):
@@ -161,5 +163,7 @@ class GeneralScorer(BaseScorer):
response = self.agent.step(
query, response_format=self.GeneralScoreSchema
)
- score_data = json.loads(response.msg.content)
- return score_data
+ parsed = safe_extract_parsed(response, self.GeneralScoreSchema)
+ if parsed is None:
+ return {"diversity": 0, "complexity": 0, "validity": 0}
+ return parsed.model_dump()
diff --git a/camel/societies/workforce/workforce.py b/camel/societies/workforce/workforce.py
index b758a4b44..9797e8566 100644
--- a/camel/societies/workforce/workforce.py
+++ b/camel/societies/workforce/workforce.py
@@ -91,7 +91,7 @@ from camel.toolkits import (
SearchToolkit,
ThinkingToolkit,
)
-from camel.utils import dependencies_required
+from camel.utils import dependencies_required, safe_extract_parsed
from .events import (
AllTasksCompletedEvent,
@@ -1624,7 +1624,14 @@ class Workforce(BaseNode):
response = self.task_agent.step(
analysis_prompt, response_format=result_schema
)
- return response.msg.parsed
+ parsed = safe_extract_parsed(response, result_schema)
+ if parsed is not None:
+ return parsed
+ logger.warning(
+ "Failed to get structured response from model, "
+ "using fallback values"
+ )
+ return TaskAnalysisResult(**fallback_values)
except Exception as e:
logger.warning(
diff --git a/camel/utils/__init__.py b/camel/utils/__init__.py
index 0fc83f68a..f285f2dc3 100644
--- a/camel/utils/__init__.py
+++ b/camel/utils/__init__.py
@@ -37,6 +37,7 @@ from .commons import (
print_text_animated,
retry_on_error,
run_async,
+ safe_extract_parsed,
text_extract_from_web,
to_pascal,
track_agent,
@@ -107,6 +108,7 @@ __all__ = [
"sanitize_filename",
"browser_toolkit_save_auth_cookie",
"run_async",
+ "safe_extract_parsed",
"configure_langfuse",
"is_langfuse_available",
"get_current_agent_session_id",
diff --git a/camel/utils/commons.py b/camel/utils/commons.py
index 37d96eba0..5b8e9ee48 100644
--- a/camel/utils/commons.py
+++ b/camel/utils/commons.py
@@ -28,6 +28,7 @@ from functools import wraps
from http import HTTPStatus
from pathlib import Path
from typing import (
+ TYPE_CHECKING,
Any,
Callable,
Dict,
@@ -40,6 +41,9 @@ from typing import (
TypeVar,
cast,
)
+
+if TYPE_CHECKING:
+ from camel.responses.agent_responses import ChatAgentResponse
from urllib.parse import urlparse
import pydantic
@@ -51,6 +55,7 @@ from camel.types import TaskType
from .constants import Constants
F = TypeVar('F', bound=Callable[..., Any])
+T = TypeVar('T', bound=BaseModel)
logger = logging.getLogger(__name__)
@@ -702,7 +707,7 @@ def retry_on_error(
except Exception as e:
last_exception = e
if attempt == max_retries:
- logger.error(
+ logger.warning(
f"Failed after {max_retries} retries: {e!s}"
)
raise
@@ -959,6 +964,65 @@ def generate_prompt_for_structured_output(
return final_prompt
+def safe_extract_parsed(
+ response: "ChatAgentResponse",
+ schema: Type[T],
+) -> Optional[T]:
+ r"""Safely extract a parsed structured output from a ChatAgentResponse.
+
+ Handles the common cases where ``response.msg`` is ``None`` (empty or
+ multi-message response) or ``msg.parsed`` is ``None`` (model failed to
+ produce valid structured output). When the parsed value is a dict, it
+ attempts to construct the schema from it.
+
+ Args:
+ response (ChatAgentResponse): The agent response to extract from.
+ schema (Type[T]): The expected Pydantic model class.
+
+ Returns:
+ Optional[T]: The parsed and validated result, or ``None`` if
+ extraction fails for any reason.
+ """
+ msg = response.msg
+ # Empty or multi-message response (len(msgs) != 1)
+ if msg is None:
+ logger.error(
+ f"safe_extract_parsed: response.msg is None "
+ f"(msgs count: {len(response.msgs)}), "
+ f"cannot extract {schema.__name__}"
+ )
+ return None
+ parsed = msg.parsed
+ # Already the expected Pydantic model
+ if isinstance(parsed, schema):
+ return parsed
+ # TODO: Unify backend parsing so parsed is always a model instance,
+ # removing the need for this dict fallback.
+ # LiteLLM or non-OpenAI backends may return raw dict instead of model
+ if isinstance(parsed, dict):
+ try:
+ return schema(**parsed)
+ except Exception as e:
+ logger.error(
+ f"safe_extract_parsed: failed to construct "
+ f"{schema.__name__} from dict: {e}"
+ )
+ return None
+ # Model did not produce structured output
+ if parsed is None:
+ logger.error(
+ f"safe_extract_parsed: msg.parsed is None, "
+ f"model did not produce valid {schema.__name__}"
+ )
+ # Unexpected type
+ else:
+ logger.error(
+ f"safe_extract_parsed: msg.parsed is "
+ f"{type(parsed).__name__}, expected {schema.__name__}"
+ )
+ return None
+
+
def with_timeout(timeout=None):
r"""Decorator that adds timeout functionality to functions.
diff --git a/test/utils/test_commons.py b/test/utils/test_commons.py
index 6088da9b0..3586d68df 100644
--- a/test/utils/test_commons.py
+++ b/test/utils/test_commons.py
@@ -17,6 +17,7 @@ from unittest import TestCase
from unittest.mock import patch
import pytest
+from pydantic import BaseModel
from camel.toolkits import FunctionTool
from camel.toolkits.base import BaseToolkit
@@ -439,3 +440,75 @@ def test_timeout_preserves_traceback():
assert "deepest_function" in tb_str
assert "inner_function" in tb_str
assert "Error from deepest function" in str(exc_info.value)
+
+
+# --- Tests for safe_extract_parsed ---
+
+
+class _DummySchema(BaseModel):
+ """Minimal Pydantic schema for safe_extract_parsed tests."""
+
+ value: str
+ score: int = 0
+
+
+def _make_response(parsed_value, num_msgs=1):
+ """Build a ChatAgentResponse with a controlled parsed value."""
+ from camel.messages.base import BaseMessage
+ from camel.responses.agent_responses import ChatAgentResponse
+ from camel.types import RoleType
+
+ if num_msgs == 0:
+ return ChatAgentResponse(msgs=[], terminated=False, info={})
+
+ msg = BaseMessage(
+ role_name="assistant",
+ role_type=RoleType.ASSISTANT,
+ meta_dict=None,
+ content="mock",
+ parsed=parsed_value,
+ )
+ return ChatAgentResponse(msgs=[msg], terminated=False, info={})
+
+
+def test_safe_extract_parsed_returns_valid_instance():
+ from camel.utils import safe_extract_parsed
+
+ expected = _DummySchema(value="hello", score=42)
+ response = _make_response(expected)
+ result = safe_extract_parsed(response, _DummySchema)
+ assert result is expected
+
+
+def test_safe_extract_parsed_constructs_from_dict():
+ from camel.utils import safe_extract_parsed
+
+ response = _make_response({"value": "from_dict", "score": 7})
+ result = safe_extract_parsed(response, _DummySchema)
+ assert isinstance(result, _DummySchema)
+ assert result.value == "from_dict"
+ assert result.score == 7
+
+
+def test_safe_extract_parsed_returns_none_for_invalid_dict():
+ from camel.utils import safe_extract_parsed
+
+ response = _make_response({"bad_key": "nope"})
+ result = safe_extract_parsed(response, _DummySchema)
+ assert result is None
+
+
+def test_safe_extract_parsed_returns_none_when_msg_is_none():
+ from camel.utils import safe_extract_parsed
+
+ response = _make_response(None, num_msgs=0)
+ result = safe_extract_parsed(response, _DummySchema)
+ assert result is None
+
+
+def test_safe_extract_parsed_returns_none_when_parsed_is_none():
+ from camel.utils import safe_extract_parsed
+
+ response = _make_response(None)
+ result = safe_extract_parsed(response, _DummySchema)
+ assert result is None
diff --git a/test/workforce/test_workforce.py b/test/workforce/test_workforce.py
index a73feb570..95adb51ce 100644
--- a/test/workforce/test_workforce.py
+++ b/test/workforce/test_workforce.py
@@ -19,8 +19,15 @@ from unittest.mock import AsyncMock, MagicMock, patch
import pytest
from camel.agents import ChatAgent
+from camel.messages.base import BaseMessage
from camel.models import ModelFactory
+from camel.responses.agent_responses import ChatAgentResponse
from camel.societies.workforce.task_channel import TaskChannel
+from camel.societies.workforce.utils import (
+ FailureHandlingConfig,
+ RecoveryStrategy,
+ TaskAnalysisResult,
+)
from camel.societies.workforce.workforce import Workforce
from camel.tasks.task import Task, TaskState
from camel.types import ModelPlatformType, ModelType
@@ -492,3 +499,177 @@ def test_start_child_node_helper():
workforce._state = WorkforceState.PAUSED
workforce._loop = None
workforce._start_child_node_when_paused(mock_coroutine())
+
+
+# --- Tests for _analyze_task null-safety (use_structured_output_handler=False)
+
+
+def _make_analyze_task_workforce():
+ r"""Create a Workforce with mocked internals for _analyze_task testing.
+
+ Bypasses API-key-dependent initialization entirely by using
+ object.__new__ and setting only the attributes that _analyze_task needs.
+ Sets use_structured_output_handler=False to test the native structured
+ output code path.
+ """
+ workforce = object.__new__(Workforce)
+ workforce.use_structured_output_handler = False
+ workforce.failure_handling_config = FailureHandlingConfig()
+
+ mock_task_agent = MagicMock()
+ mock_task_agent.reset = MagicMock()
+ workforce.task_agent = mock_task_agent
+ return workforce
+
+
+def _make_task():
+ r"""Create a minimal Task for _analyze_task tests."""
+ task = MagicMock(spec=Task)
+ task.id = "test_task_1"
+ task.content = "Test task content"
+ task.result = "Test result"
+ task.failure_count = 0
+ task.get_depth.return_value = 0
+ task.assigned_worker_id = "worker_1"
+ return task
+
+
+def _make_mock_response(parsed_value, num_msgs=1):
+ r"""Create a ChatAgentResponse with a controlled parsed value.
+
+ Uses a real BaseMessage to satisfy Pydantic validation in
+ ChatAgentResponse, with the `parsed` field set to the desired value.
+ """
+ from camel.types import RoleType
+
+ if num_msgs == 0:
+ return ChatAgentResponse(msgs=[], terminated=False, info={})
+
+ msg = BaseMessage(
+ role_name="assistant",
+ role_type=RoleType.ASSISTANT,
+ meta_dict=None,
+ content="mock content",
+ parsed=parsed_value,
+ )
+ return ChatAgentResponse(msgs=[msg], terminated=False, info={})
+
+
+def test_analyze_task_returns_valid_result():
+ r"""Test that _analyze_task returns a valid TaskAnalysisResult when
+ response.msg.parsed is a proper TaskAnalysisResult."""
+ workforce = _make_analyze_task_workforce()
+ task = _make_task()
+
+ expected = TaskAnalysisResult(
+ reasoning="Network timeout, retrying",
+ recovery_strategy=RecoveryStrategy.RETRY,
+ issues=["timeout"],
+ )
+
+ workforce.task_agent.step.return_value = _make_mock_response(expected)
+
+ result = workforce._analyze_task(
+ task, for_failure=True, error_message="timeout"
+ )
+
+ assert isinstance(result, TaskAnalysisResult)
+ assert result.recovery_strategy == RecoveryStrategy.RETRY
+ assert result.reasoning == "Network timeout, retrying"
+
+
+def test_analyze_task_fallback_when_msg_is_none():
+ r"""Test that _analyze_task falls back when response.msg is None
+ (e.g., empty msgs list)."""
+ workforce = _make_analyze_task_workforce()
+ task = _make_task()
+
+ workforce.task_agent.step.return_value = _make_mock_response(
+ None, num_msgs=0
+ )
+
+ result = workforce._analyze_task(
+ task, for_failure=True, error_message="some error"
+ )
+
+ assert isinstance(result, TaskAnalysisResult)
+ assert result.recovery_strategy == RecoveryStrategy.RETRY
+ assert "some error" in result.issues
+
+
+def test_analyze_task_fallback_when_parsed_is_none():
+ r"""Test that _analyze_task falls back when response.msg.parsed is None
+ (model failed to produce structured output)."""
+ workforce = _make_analyze_task_workforce()
+ task = _make_task()
+
+ workforce.task_agent.step.return_value = _make_mock_response(None)
+
+ result = workforce._analyze_task(
+ task, for_failure=True, error_message="parse failure"
+ )
+
+ assert isinstance(result, TaskAnalysisResult)
+ assert result.recovery_strategy == RecoveryStrategy.RETRY
+
+
+def test_analyze_task_constructs_from_dict():
+ r"""Test that _analyze_task constructs a TaskAnalysisResult when
+ response.msg.parsed is a dict."""
+ workforce = _make_analyze_task_workforce()
+ task = _make_task()
+
+ workforce.task_agent.step.return_value = _make_mock_response(
+ {
+ "reasoning": "Dict-based result",
+ "recovery_strategy": RecoveryStrategy.REPLAN,
+ "issues": ["incomplete"],
+ }
+ )
+
+ result = workforce._analyze_task(
+ task, for_failure=True, error_message="incomplete"
+ )
+
+ assert isinstance(result, TaskAnalysisResult)
+ assert result.recovery_strategy == RecoveryStrategy.REPLAN
+ assert result.reasoning == "Dict-based result"
+
+
+def test_analyze_task_fallback_when_dict_is_invalid():
+ r"""Test that _analyze_task falls back when response.msg.parsed is a dict
+ with invalid keys that fail schema validation."""
+ workforce = _make_analyze_task_workforce()
+ task = _make_task()
+
+ workforce.task_agent.step.return_value = _make_mock_response(
+ {
+ "bad_key": "unexpected value",
+ "another_bad_key": 123,
+ }
+ )
+
+ result = workforce._analyze_task(
+ task, for_failure=True, error_message="bad dict"
+ )
+
+ assert isinstance(result, TaskAnalysisResult)
+ assert result.recovery_strategy == RecoveryStrategy.RETRY
+ assert "bad dict" in result.issues
+
+
+def test_analyze_task_quality_eval_fallback():
+ r"""Test that _analyze_task returns quality evaluation fallback when
+ response.msg is None and for_failure=False."""
+ workforce = _make_analyze_task_workforce()
+ task = _make_task()
+
+ workforce.task_agent.step.return_value = _make_mock_response(
+ None, num_msgs=0
+ )
+
+ result = workforce._analyze_task(task, for_failure=False)
+
+ assert isinstance(result, TaskAnalysisResult)
+ assert result.quality_score == 80
+ assert result.recovery_strategy is None