Framework for orchestrating role-playing, autonomous AI agents. By fostering collaborative intelligence, CrewAI empowers agents to work together seamlessly, tackling complex tasks.
[OSS-151] Fail closed when human-feedback emit cannot classify (#7188)
Vidit Ostwal8e4620561963
diff --git a/lib/crewai/src/crewai/flow/dsl/_human_feedback.py b/lib/crewai/src/crewai/flow/dsl/_human_feedback.pyindex 1236ba6f7..9b13cd705 100644--- a/lib/crewai/src/crewai/flow/dsl/_human_feedback.py+++ b/lib/crewai/src/crewai/flow/dsl/_human_feedback.py@@ -23,7 +23,7 @@ __all__ = ["HumanFeedbackResult", "human_feedback"]def human_feedback(message: str,emit: Sequence[str] | None = None,- llm: str | BaseLLM | None = "gpt-5.4-mini",+ llm: str | BaseLLM | None = None,default_outcome: str | None = None,metadata: dict[str, Any] | None = None,provider: HumanFeedbackProvider | None = None,@@ -37,9 +37,7 @@ def human_feedback(configuration on the method, and the Flow engine collects and routesfeedback after the method completes, driven by the flow's definition."""- _validate_human_feedback_options(- emit=emit, llm=llm, default_outcome=default_outcome- )+ _validate_human_feedback_options(emit=emit, default_outcome=default_outcome)config = HumanFeedbackConfig(message=message,emit=list(emit) if emit is not None else None,diff --git a/lib/crewai/src/crewai/flow/flow_definition.py b/lib/crewai/src/crewai/flow/flow_definition.pyindex c55b437c2..cad4da79b 100644--- a/lib/crewai/src/crewai/flow/flow_definition.py+++ b/lib/crewai/src/crewai/flow/flow_definition.py@@ -295,8 +295,12 @@ class FlowHumanFeedbackDefinition(BaseModel):examples=[["approved", "revise"]],)llm: Any = Field(- default="gpt-4o-mini",- description="LLM configuration used to assist or process human feedback.",+ default=None,+ description=(+ "LLM used to collapse feedback to an emit outcome. "+ "None resolves at runtime via create_llm (project MODEL env, "+ "then DEFAULT_LLM_MODEL)."+ ),examples=["gpt-4o-mini"],)default_outcome: str | None = Field(@@ -1006,14 +1010,6 @@ def log_flow_definition_issues(definition: FlowDefinition) -> None:)if method.human_feedback:human_feedback_config = method.human_feedback- if human_feedback_config.emit and not human_feedback_config.llm:- _log_flow_definition_issue(- definition.name,- code="human_feedback_llm_required",- severity="error",- path=f"{path}.human_feedback.llm",- message="llm is required when human_feedback.emit is set",- )if (human_feedback_config.default_outcome is not Noneand not human_feedback_config.emitdiff --git a/lib/crewai/src/crewai/flow/human_feedback.py b/lib/crewai/src/crewai/flow/human_feedback.pyindex 0e0785f99..dd107604c 100644--- a/lib/crewai/src/crewai/flow/human_feedback.py+++ b/lib/crewai/src/crewai/flow/human_feedback.py@@ -77,7 +77,37 @@ logger = logging.getLogger(__name__)F = TypeVar("F", bound=Callable[..., Any])-__all__ = ["HumanFeedbackResult", "human_feedback"]+__all__ = ["HumanFeedbackCollapseError", "HumanFeedbackResult", "human_feedback"]+++class HumanFeedbackCollapseError(ValueError):+ """Raised when human feedback cannot be mapped to an emit outcome."""+++def _match_collapse_outcome(response_text: str, outcomes: Sequence[str]) -> str | None:+ """Return the emit label that best matches ``response_text``, or None."""+ response_clean = response_text.strip()+ for outcome in outcomes:+ if outcome.lower() == response_clean.lower():+ return outcome+ response_lower = response_clean.lower()+ best_outcome: str | None = None+ best_len = -1+ for outcome in outcomes:+ if outcome.lower() in response_lower and len(outcome) > best_len:+ best_outcome = outcome+ best_len = len(outcome)+ return best_outcome+++def _require_collapse_outcome(response_text: str, outcomes: Sequence[str]) -> str:+ matched = _match_collapse_outcome(response_text, outcomes)+ if matched is None:+ raise HumanFeedbackCollapseError(+ f"Could not match LLM response {response_text!r} to outcomes "+ f"{list(outcomes)}."+ )+ return matcheddef _serialize_llm_for_context(llm: Any) -> dict[str, Any] | str | None:@@ -165,7 +195,10 @@ class HumanFeedbackConfig:Attributes:message: The message shown to the human when requesting feedback.emit: Optional sequence of outcome strings for routing.- llm: The LLM model to use for collapsing feedback to outcomes.+ llm: The LLM used to collapse feedback to an emit outcome.+ None means resolve at runtime via create_llm (decorator+ value, then MODEL / MODEL_NAME / OPENAI_MODEL_NAME, then+ DEFAULT_LLM_MODEL).default_outcome: The outcome to use when no feedback is provided.metadata: Optional metadata for enterprise integrations.provider: Optional custom feedback provider for async workflows.@@ -173,7 +206,7 @@ class HumanFeedbackConfig:message: stremit: Sequence[str] | None = None- llm: str | BaseLLM | None = "gpt-5.4-mini"+ llm: str | BaseLLM | None = Nonedefault_outcome: str | None = Nonemetadata: dict[str, Any] | None = Noneprovider: HumanFeedbackProvider | None = None@@ -205,17 +238,9 @@ class DistilledLessons(BaseModel):def _validate_human_feedback_options(emit: Sequence[str] | None,- llm: Any,default_outcome: str | None,) -> None:if emit is not None:- if not llm:- raise ValueError(- "llm is required when emit is specified. "- "Provide an LLM model string (e.g., 'gpt-5.4-mini') or a BaseLLM instance. "- "See the CrewAI Human-in-the-Loop (HITL) documentation for more information: "- "https://docs.crewai.com/en/learn/human-feedback-in-flows"- )if default_outcome is not None and default_outcome not in emit:raise ValueError(f"default_outcome '{default_outcome}' must be one of the "@@ -232,16 +257,23 @@ def _get_hitl_prompt(key: str) -> str:def _resolve_llm_instance(llm: Any) -> Any:+ """Resolve a collapse/learn LLM the same way agents resolve theirs.++ Explicit decorator values win. ``None`` follows ``create_llm``:+ ``MODEL`` / ``MODEL_NAME`` / ``OPENAI_MODEL_NAME``, then+ ``DEFAULT_LLM_MODEL``.+ """from crewai.llm import LLM+ from crewai.utilities.llm_utils import create_llmif llm is None:- return LLM(model="gpt-5.4-mini")+ return create_llm(None)if isinstance(llm, str):return LLM(model=llm)if isinstance(llm, dict):deserialized = _deserialize_llm_from_context(llm)- return deserialized if deserialized is not None else LLM(model="gpt-5.4-mini")- return llm # already a BaseLLM instance+ return deserialized if deserialized is not None else create_llm(None)+ return llmdef _pre_review_with_lessons(@@ -370,7 +402,7 @@ def _distill_and_store_lessons(def human_feedback(message: str,emit: Sequence[str] | None = None,- llm: str | BaseLLM | None = "gpt-5.4-mini",+ llm: str | BaseLLM | None = None,default_outcome: str | None = None,metadata: dict[str, Any] | None = None,provider: HumanFeedbackProvider | None = None,diff --git a/lib/crewai/src/crewai/flow/runtime/__init__.py b/lib/crewai/src/crewai/flow/runtime/__init__.pyindex 63d9630d6..e1f7e5f8f 100644--- a/lib/crewai/src/crewai/flow/runtime/__init__.py+++ b/lib/crewai/src/crewai/flow/runtime/__init__.py@@ -110,10 +110,12 @@ from crewai.flow.flow_wrappers import (StartMethod,)from crewai.flow.human_feedback import (+ HumanFeedbackCollapseError,HumanFeedbackResult,- _deserialize_llm_from_context,_distill_and_store_lessons,_pre_review_with_lessons,+ _require_collapse_outcome,+ _resolve_llm_instance,_serialize_llm_for_context,)from crewai.flow.input_provider import InputProvider@@ -3613,13 +3615,11 @@ class Flow(BaseModel, Generic[T], metaclass=FlowMeta):method_output: Any,) -> Any:llm = feedback_definition.llm- llm_instance = (- _deserialize_llm_from_context(llm) if isinstance(llm, (str, dict)) else llm- )emit = feedback_definition.emitdefault_outcome = feedback_definition.default_outcomemetadata = feedback_definition.metadatalearn = feedback_definition.learn and self.memory is not None+ llm_instance = _resolve_llm_instance(llm) if (emit or learn) else llmif learn:method_output = await asyncio.to_thread(@@ -3712,22 +3712,23 @@ class Flow(BaseModel, Generic[T], metaclass=FlowMeta):elif emit:collapsed_outcome = emit[0]elif emit:- collapse_llm = (- _deserialize_llm_from_context(llm)- if isinstance(llm, (str, dict))- else llm- )- if collapse_llm is not None:- collapsed_outcome = await asyncio.to_thread(- self._collapse_to_outcome,- feedback=raw_feedback,- outcomes=emit,- llm=collapse_llm,+ collapse_llm = _resolve_llm_instance(llm)+ if collapse_llm is None:+ raise HumanFeedbackCollapseError(+ "Could not resolve an LLM to classify human feedback. "+ "Set llm= on @human_feedback or MODEL / MODEL_NAME / "+ "OPENAI_MODEL_NAME.")- else:- collapsed_outcome = emit[0]+ collapsed_outcome = await asyncio.to_thread(+ self._collapse_to_outcome,+ feedback=raw_feedback,+ outcomes=emit,+ llm=collapse_llm,+ )if emit and collapsed_outcome is None:- collapsed_outcome = default_outcome or emit[0]+ raise HumanFeedbackCollapseError(+ f"Could not classify human feedback into one of {list(emit)}."+ )result = HumanFeedbackResult(output=method_output,@@ -3849,6 +3850,10 @@ class Flow(BaseModel, Generic[T], metaclass=FlowMeta):Returns:One of the outcome strings that best matches the feedback intent.++ Raises:+ HumanFeedbackCollapseError: If the LLM cannot be called or its+ response cannot be mapped to one of ``outcomes``."""from typing import Literal@@ -3889,27 +3894,6 @@ class Flow(BaseModel, Generic[T], metaclass=FlowMeta):messages=[{"role": "user", "content": prompt}],response_model=FeedbackOutcome,)-- if isinstance(response, str):- import json-- try:- parsed = json.loads(response)- return str(parsed.get("outcome", outcomes[0]))- except json.JSONDecodeError:- response_clean = response.strip()- for outcome in outcomes:- if outcome.lower() == response_clean.lower():- return outcome- return outcomes[0]- elif isinstance(response, FeedbackOutcome):- return str(response.outcome)- elif hasattr(response, "outcome"):- return str(response.outcome)- else:- logger.warning(f"Unexpected response type: {type(response)}")- return outcomes[0]-except HookAborted:raiseexcept Exception as e:@@ -3920,37 +3904,34 @@ class Flow(BaseModel, Generic[T], metaclass=FlowMeta):response = llm_instance.call(messages=[{"role": "user", "content": prompt}],)- response_clean = str(response).strip()-- for outcome in outcomes:- if outcome.lower() == response_clean.lower():- return outcome-- # Partial match (longest wins, first on length ties)- response_lower = response_clean.lower()- best_outcome: str | None = None- best_len = -1- for outcome in outcomes:- if outcome.lower() in response_lower and len(outcome) > best_len:- best_outcome = outcome- best_len = len(outcome)- if best_outcome is not None:- return best_outcome-- logger.warning(- f"Could not match LLM response '{response_clean}' to outcomes {list(outcomes)}. "- f"Falling back to first outcome: {outcomes[0]}"- )- return outcomes[0]-except HookAborted:raiseexcept Exception as fallback_err:- logger.warning(- f"Simple prompting also failed: {fallback_err}. "- f"Falling back to first outcome: {outcomes[0]}"- )- return outcomes[0]+ raise HumanFeedbackCollapseError(+ f"Could not classify human feedback into {list(outcomes)}: "+ f"{fallback_err}"+ ) from fallback_err+ return _require_collapse_outcome(str(response), outcomes)++ if isinstance(response, str):+ import json++ try:+ parsed = json.loads(response)+ except json.JSONDecodeError:+ return _require_collapse_outcome(response, outcomes)+ if isinstance(parsed, dict):+ outcome = parsed.get("outcome")+ if isinstance(outcome, str):+ return _require_collapse_outcome(outcome, outcomes)+ return _require_collapse_outcome(response, outcomes)+ if isinstance(response, FeedbackOutcome):+ return str(response.outcome)+ if hasattr(response, "outcome"):+ return _require_collapse_outcome(str(response.outcome), outcomes)+ raise HumanFeedbackCollapseError(+ f"Unexpected collapse response type: {type(response)}"+ )def _log_flow_event(self,diff --git a/lib/crewai/tests/test_async_human_feedback.py b/lib/crewai/tests/test_async_human_feedback.pyindex 174d2cc26..8fba9d113 100644--- a/lib/crewai/tests/test_async_human_feedback.py+++ b/lib/crewai/tests/test_async_human_feedback.py@@ -1099,6 +1099,42 @@ class TestCollapseToOutcomeJsonParsing:assert result == "approved"+ def test_json_without_outcome_raises(self) -> None:+ """Parsed JSON missing a usable outcome fails closed."""+ from crewai.flow.human_feedback import HumanFeedbackCollapseError++ flow = Flow()++ with patch("crewai.llm.LLM") as MockLLM:+ mock_llm = MagicMock()+ mock_llm.call.return_value = '{"foo": "bar"}'+ MockLLM.return_value = mock_llm++ with pytest.raises(HumanFeedbackCollapseError, match="Could not match"):+ flow._collapse_to_outcome(+ feedback="looks good",+ outcomes=["approved", "rejected"],+ llm="gpt-4o-mini",+ )++ def test_non_object_json_does_not_attribute_error(self) -> None:+ """JSON that is not an object is matched as raw text, not parsed.get."""+ from crewai.flow.human_feedback import HumanFeedbackCollapseError++ flow = Flow()++ with patch("crewai.llm.LLM") as MockLLM:+ mock_llm = MagicMock()+ mock_llm.call.return_value = "null"+ MockLLM.return_value = mock_llm++ with pytest.raises(HumanFeedbackCollapseError, match="Could not match"):+ flow._collapse_to_outcome(+ feedback="looks good",+ outcomes=["approved", "rejected"],+ llm="gpt-4o-mini",+ )+def test_llm_exception_falls_back_to_simple_prompting(self) -> None:"""Test that LLM exception triggers fallback to simple prompting."""flow = Flow()diff --git a/lib/crewai/tests/test_human_feedback_decorator.py b/lib/crewai/tests/test_human_feedback_decorator.pyindex d3dac0a22..17557c7e2 100644--- a/lib/crewai/tests/test_human_feedback_decorator.py+++ b/lib/crewai/tests/test_human_feedback_decorator.py@@ -13,29 +13,45 @@ from unittest.mock import MagicMock, patchimport pytest+from crewai.constants import DEFAULT_LLM_MODELfrom crewai.flow import Flow, human_feedback, listen, persist, startfrom crewai.flow.human_feedback import (+ HumanFeedbackCollapseError,HumanFeedbackConfig,HumanFeedbackResult,+ _resolve_llm_instance,)class TestHumanFeedbackValidation:"""Tests for decorator parameter validation."""- def test_emit_requires_llm(self):- """Test that specifying emit with llm=None raises ValueError."""- with pytest.raises(ValueError) as exc_info:+ def test_emit_allows_omitted_llm(self):+ """emit without an explicit llm is allowed; the engine resolves one later."""- @human_feedback(- message="Review this:",- emit=["approve", "reject"],- llm=None,- )- def test_method(self):- return "output"+ @human_feedback(+ message="Review this:",+ emit=["approve", "reject"],+ )+ def test_method(self):+ return "output"- assert "llm is required" in str(exc_info.value)+ config = test_method.__human_feedback_config__+ assert config.emit == ["approve", "reject"]+ assert config.llm is None++ def test_emit_allows_explicit_llm_none(self):+ """llm=None with emit is the same as omitting llm."""++ @human_feedback(+ message="Review this:",+ emit=["approve", "reject"],+ llm=None,+ )+ def test_method(self):+ return "output"++ assert test_method.__human_feedback_config__.llm is Nonedef test_default_outcome_requires_emit(self):"""Test that specifying default_outcome without emit raises ValueError."""@@ -129,6 +145,38 @@ class TestHumanFeedbackConfig:assert config.metadata == {"key": "value"}+class TestResolveLlmInstance:+ """Omitted decorator llm follows create_llm (project MODEL, then default)."""++ def test_none_uses_model_env(self, monkeypatch: pytest.MonkeyPatch):+ monkeypatch.setenv("MODEL", "anthropic/claude-sonnet-4-5")+ monkeypatch.delenv("MODEL_NAME", raising=False)+ monkeypatch.delenv("OPENAI_MODEL_NAME", raising=False)++ llm = _resolve_llm_instance(None)++ assert llm is not None+ assert "claude-sonnet-4-5" in getattr(llm, "model", "")++ def test_explicit_string_wins_over_model_env(self, monkeypatch: pytest.MonkeyPatch):+ monkeypatch.setenv("MODEL", "anthropic/claude-sonnet-4-5")++ llm = _resolve_llm_instance("gpt-4o-mini")++ assert llm is not None+ assert getattr(llm, "model", "") == "gpt-4o-mini"++ def test_none_falls_back_to_default_model(self, monkeypatch: pytest.MonkeyPatch):+ monkeypatch.delenv("MODEL", raising=False)+ monkeypatch.delenv("MODEL_NAME", raising=False)+ monkeypatch.delenv("OPENAI_MODEL_NAME", raising=False)++ llm = _resolve_llm_instance(None)++ assert llm is not None+ assert getattr(llm, "model", "") == DEFAULT_LLM_MODEL++class TestHumanFeedbackResult:"""Tests for HumanFeedbackResult dataclass."""@@ -318,6 +366,41 @@ class TestHumanFeedbackExecution:# But the outcome is still correctly set for routing purposesassert flow.last_human_feedback.outcome == "approved"+ @patch("builtins.input", return_value="Approved!")+ @patch("builtins.print")+ def test_omitted_llm_collapse_uses_project_model(+ self, mock_print, mock_input, monkeypatch: pytest.MonkeyPatch+ ):+ """When emit is set without llm=, collapse uses MODEL via create_llm."""+ monkeypatch.setenv("MODEL", "anthropic/claude-sonnet-4-5")+ monkeypatch.delenv("MODEL_NAME", raising=False)+ monkeypatch.delenv("OPENAI_MODEL_NAME", raising=False)++ class TestFlow(Flow):+ @start()+ @human_feedback(+ message="Review:",+ emit=["approved", "rejected"],+ )+ def review(self):+ return "Content"++ flow = TestFlow()++ with (+ patch.object(+ flow, "_request_human_feedback", return_value="Looks great, approved!"+ ),+ patch.object(flow, "_collapse_to_outcome", return_value="approved") as mock_collapse,+ ):+ result = flow.kickoff()++ assert result == "Content"+ mock_collapse.assert_called_once()+ collapse_llm = mock_collapse.call_args.kwargs["llm"]+ assert collapse_llm is not None+ assert "claude-sonnet-4-5" in getattr(collapse_llm, "model", "")+class TestHumanFeedbackHistory:"""Tests for human feedback history tracking."""@@ -405,8 +488,8 @@ class TestCollapseToOutcome:assert result == "approved"- def test_fallback_to_first(self):- """Test that unmatched response falls back to first outcome."""+ def test_unmatched_response_raises(self):+ """Unmatched LLM text fails closed instead of taking emit[0]."""flow = Flow()with patch("crewai.llm.LLM") as MockLLM:@@ -414,31 +497,28 @@ class TestCollapseToOutcome:mock_llm.call.return_value = "something completely different"MockLLM.return_value = mock_llm- result = flow._collapse_to_outcome(- feedback="Unclear feedback",- outcomes=["approved", "rejected"],- llm="gpt-4o-mini",- )-- assert result == "approved"+ with pytest.raises(HumanFeedbackCollapseError, match="Could not match"):+ flow._collapse_to_outcome(+ feedback="Unclear feedback",+ outcomes=["approved", "rejected"],+ llm="gpt-4o-mini",+ )- def test_both_llm_calls_fail_returns_first_outcome(self):- """When both structured and simple prompting fail, return outcomes[0]."""+ def test_both_llm_calls_fail_raises(self):+ """When both structured and simple prompting fail, raise instead of emit[0]."""flow = Flow()with patch("crewai.llm.LLM") as MockLLM:mock_llm = MagicMock()- # Both calls raise — simulates wrong provider / auth failuremock_llm.call.side_effect = RuntimeError("Model not found")MockLLM.return_value = mock_llm- result = flow._collapse_to_outcome(- feedback="looks great, approve it",- outcomes=["needs_changes", "approved"],- llm="gemini-3-flash-preview",- )-- assert result == "needs_changes" # First in list (safe fallback)+ with pytest.raises(HumanFeedbackCollapseError, match="Could not classify"):+ flow._collapse_to_outcome(+ feedback="looks great, approve it",+ outcomes=["needs_changes", "approved"],+ llm="gemini-3-flash-preview",+ )def test_structured_fails_but_simple_succeeds(self):"""When structured output fails but simple prompting works, use that."""@@ -460,7 +540,70 @@ class TestCollapseToOutcome:assert result == "approved"+ def test_collapse_failure_does_not_route_to_first_emit(self):+ """A failed collapse must not fire the first emit listener."""+ routed: list[str] = []++ class TestFlow(Flow):+ @start()+ @human_feedback(+ message="Review:",+ emit=["approved", "rejected"],+ llm="gpt-4o-mini",+ )+ def review(self):+ return "payment of $10"++ @listen("approved")+ def send_money(self):+ routed.append("approved")+ return "sent"++ @listen("rejected")+ def stop(self):+ routed.append("rejected")+ return "stopped"+ flow = TestFlow()+ with (+ patch.object(+ flow, "_request_human_feedback", return_value="no, reject this"+ ),+ patch.object(+ flow,+ "_collapse_to_outcome",+ side_effect=HumanFeedbackCollapseError("failed"),+ ),+ ):+ with pytest.raises(HumanFeedbackCollapseError, match="failed"):+ flow.kickoff()++ assert routed == []++ def test_unresolved_collapse_llm_raises(self):+ """If no LLM can be resolved, do not take emit[0]."""++ class TestFlow(Flow):+ @start()+ @human_feedback(+ message="Review:",+ emit=["approved", "rejected"],+ )+ def review(self):+ return "payment of $10"++ flow = TestFlow()+ with (+ patch.object(+ flow, "_request_human_feedback", return_value="no, reject this"+ ),+ patch(+ "crewai.flow.runtime._resolve_llm_instance",+ return_value=None,+ ),+ ):+ with pytest.raises(HumanFeedbackCollapseError, match="Could not resolve"):+ flow.kickoff()class TestHumanFeedbackLearn:@@ -596,8 +739,8 @@ class TestHumanFeedbackLearn:flow.memory.remember_many.assert_not_called()- def test_learn_true_uses_default_llm(self):- """When learn=True and llm is not explicitly set, the default gpt-5.4-mini is used."""+ def test_learn_true_omits_llm_by_default(self):+ """When learn=True and llm is not set, config.llm stays None for runtime resolve."""@human_feedback(message="Review:", learn=True)def test_method(self):@@ -606,8 +749,7 @@ class TestHumanFeedbackLearn:config = test_method.__human_feedback_config__assert config is not Noneassert config.learn is True- # llm defaults to "gpt-5.4-mini" at the function level- assert config.llm == "gpt-5.4-mini"+ assert config.llm is Nonedef test_pre_review_failure_logs_and_returns_raw_output(self, caplog):"""Pre-review LLM failure falls back to raw output AND logs a warning."""