Skip to content

Framework for orchestrating role-playing, autonomous AI agents. By fostering collaborative intelligence, CrewAI empowers agents to work together seamlessly, tackling complex tasks.

[OSS-151] Fail closed when human-feedback emit cannot classify (#7188)

Vidit Ostwal8e4620561963
diff --git a/lib/crewai/src/crewai/flow/dsl/_human_feedback.py b/lib/crewai/src/crewai/flow/dsl/_human_feedback.py
index 1236ba6f7..9b13cd705 100644
--- a/lib/crewai/src/crewai/flow/dsl/_human_feedback.py
+++ b/lib/crewai/src/crewai/flow/dsl/_human_feedback.py
@@ -23,7 +23,7 @@ __all__ = ["HumanFeedbackResult", "human_feedback"]
def human_feedback(
message: str,
emit: Sequence[str] | None = None,
- llm: str | BaseLLM | None = "gpt-5.4-mini",
+ llm: str | BaseLLM | None = None,
default_outcome: str | None = None,
metadata: dict[str, Any] | None = None,
provider: HumanFeedbackProvider | None = None,
@@ -37,9 +37,7 @@ def human_feedback(
configuration on the method, and the Flow engine collects and routes
feedback after the method completes, driven by the flow's definition.
"""
- _validate_human_feedback_options(
- emit=emit, llm=llm, default_outcome=default_outcome
- )
+ _validate_human_feedback_options(emit=emit, default_outcome=default_outcome)
config = HumanFeedbackConfig(
message=message,
emit=list(emit) if emit is not None else None,
diff --git a/lib/crewai/src/crewai/flow/flow_definition.py b/lib/crewai/src/crewai/flow/flow_definition.py
index c55b437c2..cad4da79b 100644
--- a/lib/crewai/src/crewai/flow/flow_definition.py
+++ b/lib/crewai/src/crewai/flow/flow_definition.py
@@ -295,8 +295,12 @@ class FlowHumanFeedbackDefinition(BaseModel):
examples=[["approved", "revise"]],
)
llm: Any = Field(
- default="gpt-4o-mini",
- description="LLM configuration used to assist or process human feedback.",
+ default=None,
+ description=(
+ "LLM used to collapse feedback to an emit outcome. "
+ "None resolves at runtime via create_llm (project MODEL env, "
+ "then DEFAULT_LLM_MODEL)."
+ ),
examples=["gpt-4o-mini"],
)
default_outcome: str | None = Field(
@@ -1006,14 +1010,6 @@ def log_flow_definition_issues(definition: FlowDefinition) -> None:
)
if method.human_feedback:
human_feedback_config = method.human_feedback
- if human_feedback_config.emit and not human_feedback_config.llm:
- _log_flow_definition_issue(
- definition.name,
- code="human_feedback_llm_required",
- severity="error",
- path=f"{path}.human_feedback.llm",
- message="llm is required when human_feedback.emit is set",
- )
if (
human_feedback_config.default_outcome is not None
and not human_feedback_config.emit
diff --git a/lib/crewai/src/crewai/flow/human_feedback.py b/lib/crewai/src/crewai/flow/human_feedback.py
index 0e0785f99..dd107604c 100644
--- a/lib/crewai/src/crewai/flow/human_feedback.py
+++ b/lib/crewai/src/crewai/flow/human_feedback.py
@@ -77,7 +77,37 @@ logger = logging.getLogger(__name__)
F = TypeVar("F", bound=Callable[..., Any])
-__all__ = ["HumanFeedbackResult", "human_feedback"]
+__all__ = ["HumanFeedbackCollapseError", "HumanFeedbackResult", "human_feedback"]
+
+
+class HumanFeedbackCollapseError(ValueError):
+ """Raised when human feedback cannot be mapped to an emit outcome."""
+
+
+def _match_collapse_outcome(response_text: str, outcomes: Sequence[str]) -> str | None:
+ """Return the emit label that best matches ``response_text``, or None."""
+ response_clean = response_text.strip()
+ for outcome in outcomes:
+ if outcome.lower() == response_clean.lower():
+ return outcome
+ response_lower = response_clean.lower()
+ best_outcome: str | None = None
+ best_len = -1
+ for outcome in outcomes:
+ if outcome.lower() in response_lower and len(outcome) > best_len:
+ best_outcome = outcome
+ best_len = len(outcome)
+ return best_outcome
+
+
+def _require_collapse_outcome(response_text: str, outcomes: Sequence[str]) -> str:
+ matched = _match_collapse_outcome(response_text, outcomes)
+ if matched is None:
+ raise HumanFeedbackCollapseError(
+ f"Could not match LLM response {response_text!r} to outcomes "
+ f"{list(outcomes)}."
+ )
+ return matched
def _serialize_llm_for_context(llm: Any) -> dict[str, Any] | str | None:
@@ -165,7 +195,10 @@ class HumanFeedbackConfig:
Attributes:
message: The message shown to the human when requesting feedback.
emit: Optional sequence of outcome strings for routing.
- llm: The LLM model to use for collapsing feedback to outcomes.
+ llm: The LLM used to collapse feedback to an emit outcome.
+ None means resolve at runtime via create_llm (decorator
+ value, then MODEL / MODEL_NAME / OPENAI_MODEL_NAME, then
+ DEFAULT_LLM_MODEL).
default_outcome: The outcome to use when no feedback is provided.
metadata: Optional metadata for enterprise integrations.
provider: Optional custom feedback provider for async workflows.
@@ -173,7 +206,7 @@ class HumanFeedbackConfig:
message: str
emit: Sequence[str] | None = None
- llm: str | BaseLLM | None = "gpt-5.4-mini"
+ llm: str | BaseLLM | None = None
default_outcome: str | None = None
metadata: dict[str, Any] | None = None
provider: HumanFeedbackProvider | None = None
@@ -205,17 +238,9 @@ class DistilledLessons(BaseModel):
def _validate_human_feedback_options(
emit: Sequence[str] | None,
- llm: Any,
default_outcome: str | None,
) -> None:
if emit is not None:
- if not llm:
- raise ValueError(
- "llm is required when emit is specified. "
- "Provide an LLM model string (e.g., 'gpt-5.4-mini') or a BaseLLM instance. "
- "See the CrewAI Human-in-the-Loop (HITL) documentation for more information: "
- "https://docs.crewai.com/en/learn/human-feedback-in-flows"
- )
if default_outcome is not None and default_outcome not in emit:
raise ValueError(
f"default_outcome '{default_outcome}' must be one of the "
@@ -232,16 +257,23 @@ def _get_hitl_prompt(key: str) -> str:
def _resolve_llm_instance(llm: Any) -> Any:
+ """Resolve a collapse/learn LLM the same way agents resolve theirs.
+
+ Explicit decorator values win. ``None`` follows ``create_llm``:
+ ``MODEL`` / ``MODEL_NAME`` / ``OPENAI_MODEL_NAME``, then
+ ``DEFAULT_LLM_MODEL``.
+ """
from crewai.llm import LLM
+ from crewai.utilities.llm_utils import create_llm
if llm is None:
- return LLM(model="gpt-5.4-mini")
+ return create_llm(None)
if isinstance(llm, str):
return LLM(model=llm)
if isinstance(llm, dict):
deserialized = _deserialize_llm_from_context(llm)
- return deserialized if deserialized is not None else LLM(model="gpt-5.4-mini")
- return llm # already a BaseLLM instance
+ return deserialized if deserialized is not None else create_llm(None)
+ return llm
def _pre_review_with_lessons(
@@ -370,7 +402,7 @@ def _distill_and_store_lessons(
def human_feedback(
message: str,
emit: Sequence[str] | None = None,
- llm: str | BaseLLM | None = "gpt-5.4-mini",
+ llm: str | BaseLLM | None = None,
default_outcome: str | None = None,
metadata: dict[str, Any] | None = None,
provider: HumanFeedbackProvider | None = None,
diff --git a/lib/crewai/src/crewai/flow/runtime/__init__.py b/lib/crewai/src/crewai/flow/runtime/__init__.py
index 63d9630d6..e1f7e5f8f 100644
--- a/lib/crewai/src/crewai/flow/runtime/__init__.py
+++ b/lib/crewai/src/crewai/flow/runtime/__init__.py
@@ -110,10 +110,12 @@ from crewai.flow.flow_wrappers import (
StartMethod,
)
from crewai.flow.human_feedback import (
+ HumanFeedbackCollapseError,
HumanFeedbackResult,
- _deserialize_llm_from_context,
_distill_and_store_lessons,
_pre_review_with_lessons,
+ _require_collapse_outcome,
+ _resolve_llm_instance,
_serialize_llm_for_context,
)
from crewai.flow.input_provider import InputProvider
@@ -3613,13 +3615,11 @@ class Flow(BaseModel, Generic[T], metaclass=FlowMeta):
method_output: Any,
) -> Any:
llm = feedback_definition.llm
- llm_instance = (
- _deserialize_llm_from_context(llm) if isinstance(llm, (str, dict)) else llm
- )
emit = feedback_definition.emit
default_outcome = feedback_definition.default_outcome
metadata = feedback_definition.metadata
learn = feedback_definition.learn and self.memory is not None
+ llm_instance = _resolve_llm_instance(llm) if (emit or learn) else llm
if learn:
method_output = await asyncio.to_thread(
@@ -3712,22 +3712,23 @@ class Flow(BaseModel, Generic[T], metaclass=FlowMeta):
elif emit:
collapsed_outcome = emit[0]
elif emit:
- collapse_llm = (
- _deserialize_llm_from_context(llm)
- if isinstance(llm, (str, dict))
- else llm
- )
- if collapse_llm is not None:
- collapsed_outcome = await asyncio.to_thread(
- self._collapse_to_outcome,
- feedback=raw_feedback,
- outcomes=emit,
- llm=collapse_llm,
+ collapse_llm = _resolve_llm_instance(llm)
+ if collapse_llm is None:
+ raise HumanFeedbackCollapseError(
+ "Could not resolve an LLM to classify human feedback. "
+ "Set llm= on @human_feedback or MODEL / MODEL_NAME / "
+ "OPENAI_MODEL_NAME."
)
- else:
- collapsed_outcome = emit[0]
+ collapsed_outcome = await asyncio.to_thread(
+ self._collapse_to_outcome,
+ feedback=raw_feedback,
+ outcomes=emit,
+ llm=collapse_llm,
+ )
if emit and collapsed_outcome is None:
- collapsed_outcome = default_outcome or emit[0]
+ raise HumanFeedbackCollapseError(
+ f"Could not classify human feedback into one of {list(emit)}."
+ )
result = HumanFeedbackResult(
output=method_output,
@@ -3849,6 +3850,10 @@ class Flow(BaseModel, Generic[T], metaclass=FlowMeta):
Returns:
One of the outcome strings that best matches the feedback intent.
+
+ Raises:
+ HumanFeedbackCollapseError: If the LLM cannot be called or its
+ response cannot be mapped to one of ``outcomes``.
"""
from typing import Literal
@@ -3889,27 +3894,6 @@ class Flow(BaseModel, Generic[T], metaclass=FlowMeta):
messages=[{"role": "user", "content": prompt}],
response_model=FeedbackOutcome,
)
-
- if isinstance(response, str):
- import json
-
- try:
- parsed = json.loads(response)
- return str(parsed.get("outcome", outcomes[0]))
- except json.JSONDecodeError:
- response_clean = response.strip()
- for outcome in outcomes:
- if outcome.lower() == response_clean.lower():
- return outcome
- return outcomes[0]
- elif isinstance(response, FeedbackOutcome):
- return str(response.outcome)
- elif hasattr(response, "outcome"):
- return str(response.outcome)
- else:
- logger.warning(f"Unexpected response type: {type(response)}")
- return outcomes[0]
-
except HookAborted:
raise
except Exception as e:
@@ -3920,37 +3904,34 @@ class Flow(BaseModel, Generic[T], metaclass=FlowMeta):
response = llm_instance.call(
messages=[{"role": "user", "content": prompt}],
)
- response_clean = str(response).strip()
-
- for outcome in outcomes:
- if outcome.lower() == response_clean.lower():
- return outcome
-
- # Partial match (longest wins, first on length ties)
- response_lower = response_clean.lower()
- best_outcome: str | None = None
- best_len = -1
- for outcome in outcomes:
- if outcome.lower() in response_lower and len(outcome) > best_len:
- best_outcome = outcome
- best_len = len(outcome)
- if best_outcome is not None:
- return best_outcome
-
- logger.warning(
- f"Could not match LLM response '{response_clean}' to outcomes {list(outcomes)}. "
- f"Falling back to first outcome: {outcomes[0]}"
- )
- return outcomes[0]
-
except HookAborted:
raise
except Exception as fallback_err:
- logger.warning(
- f"Simple prompting also failed: {fallback_err}. "
- f"Falling back to first outcome: {outcomes[0]}"
- )
- return outcomes[0]
+ raise HumanFeedbackCollapseError(
+ f"Could not classify human feedback into {list(outcomes)}: "
+ f"{fallback_err}"
+ ) from fallback_err
+ return _require_collapse_outcome(str(response), outcomes)
+
+ if isinstance(response, str):
+ import json
+
+ try:
+ parsed = json.loads(response)
+ except json.JSONDecodeError:
+ return _require_collapse_outcome(response, outcomes)
+ if isinstance(parsed, dict):
+ outcome = parsed.get("outcome")
+ if isinstance(outcome, str):
+ return _require_collapse_outcome(outcome, outcomes)
+ return _require_collapse_outcome(response, outcomes)
+ if isinstance(response, FeedbackOutcome):
+ return str(response.outcome)
+ if hasattr(response, "outcome"):
+ return _require_collapse_outcome(str(response.outcome), outcomes)
+ raise HumanFeedbackCollapseError(
+ f"Unexpected collapse response type: {type(response)}"
+ )
def _log_flow_event(
self,
diff --git a/lib/crewai/tests/test_async_human_feedback.py b/lib/crewai/tests/test_async_human_feedback.py
index 174d2cc26..8fba9d113 100644
--- a/lib/crewai/tests/test_async_human_feedback.py
+++ b/lib/crewai/tests/test_async_human_feedback.py
@@ -1099,6 +1099,42 @@ class TestCollapseToOutcomeJsonParsing:
assert result == "approved"
+ def test_json_without_outcome_raises(self) -> None:
+ """Parsed JSON missing a usable outcome fails closed."""
+ from crewai.flow.human_feedback import HumanFeedbackCollapseError
+
+ flow = Flow()
+
+ with patch("crewai.llm.LLM") as MockLLM:
+ mock_llm = MagicMock()
+ mock_llm.call.return_value = '{"foo": "bar"}'
+ MockLLM.return_value = mock_llm
+
+ with pytest.raises(HumanFeedbackCollapseError, match="Could not match"):
+ flow._collapse_to_outcome(
+ feedback="looks good",
+ outcomes=["approved", "rejected"],
+ llm="gpt-4o-mini",
+ )
+
+ def test_non_object_json_does_not_attribute_error(self) -> None:
+ """JSON that is not an object is matched as raw text, not parsed.get."""
+ from crewai.flow.human_feedback import HumanFeedbackCollapseError
+
+ flow = Flow()
+
+ with patch("crewai.llm.LLM") as MockLLM:
+ mock_llm = MagicMock()
+ mock_llm.call.return_value = "null"
+ MockLLM.return_value = mock_llm
+
+ with pytest.raises(HumanFeedbackCollapseError, match="Could not match"):
+ flow._collapse_to_outcome(
+ feedback="looks good",
+ outcomes=["approved", "rejected"],
+ llm="gpt-4o-mini",
+ )
+
def test_llm_exception_falls_back_to_simple_prompting(self) -> None:
"""Test that LLM exception triggers fallback to simple prompting."""
flow = Flow()
diff --git a/lib/crewai/tests/test_human_feedback_decorator.py b/lib/crewai/tests/test_human_feedback_decorator.py
index d3dac0a22..17557c7e2 100644
--- a/lib/crewai/tests/test_human_feedback_decorator.py
+++ b/lib/crewai/tests/test_human_feedback_decorator.py
@@ -13,29 +13,45 @@ from unittest.mock import MagicMock, patch
import pytest
+from crewai.constants import DEFAULT_LLM_MODEL
from crewai.flow import Flow, human_feedback, listen, persist, start
from crewai.flow.human_feedback import (
+ HumanFeedbackCollapseError,
HumanFeedbackConfig,
HumanFeedbackResult,
+ _resolve_llm_instance,
)
class TestHumanFeedbackValidation:
"""Tests for decorator parameter validation."""
- def test_emit_requires_llm(self):
- """Test that specifying emit with llm=None raises ValueError."""
- with pytest.raises(ValueError) as exc_info:
+ def test_emit_allows_omitted_llm(self):
+ """emit without an explicit llm is allowed; the engine resolves one later."""
- @human_feedback(
- message="Review this:",
- emit=["approve", "reject"],
- llm=None,
- )
- def test_method(self):
- return "output"
+ @human_feedback(
+ message="Review this:",
+ emit=["approve", "reject"],
+ )
+ def test_method(self):
+ return "output"
- assert "llm is required" in str(exc_info.value)
+ config = test_method.__human_feedback_config__
+ assert config.emit == ["approve", "reject"]
+ assert config.llm is None
+
+ def test_emit_allows_explicit_llm_none(self):
+ """llm=None with emit is the same as omitting llm."""
+
+ @human_feedback(
+ message="Review this:",
+ emit=["approve", "reject"],
+ llm=None,
+ )
+ def test_method(self):
+ return "output"
+
+ assert test_method.__human_feedback_config__.llm is None
def test_default_outcome_requires_emit(self):
"""Test that specifying default_outcome without emit raises ValueError."""
@@ -129,6 +145,38 @@ class TestHumanFeedbackConfig:
assert config.metadata == {"key": "value"}
+class TestResolveLlmInstance:
+ """Omitted decorator llm follows create_llm (project MODEL, then default)."""
+
+ def test_none_uses_model_env(self, monkeypatch: pytest.MonkeyPatch):
+ monkeypatch.setenv("MODEL", "anthropic/claude-sonnet-4-5")
+ monkeypatch.delenv("MODEL_NAME", raising=False)
+ monkeypatch.delenv("OPENAI_MODEL_NAME", raising=False)
+
+ llm = _resolve_llm_instance(None)
+
+ assert llm is not None
+ assert "claude-sonnet-4-5" in getattr(llm, "model", "")
+
+ def test_explicit_string_wins_over_model_env(self, monkeypatch: pytest.MonkeyPatch):
+ monkeypatch.setenv("MODEL", "anthropic/claude-sonnet-4-5")
+
+ llm = _resolve_llm_instance("gpt-4o-mini")
+
+ assert llm is not None
+ assert getattr(llm, "model", "") == "gpt-4o-mini"
+
+ def test_none_falls_back_to_default_model(self, monkeypatch: pytest.MonkeyPatch):
+ monkeypatch.delenv("MODEL", raising=False)
+ monkeypatch.delenv("MODEL_NAME", raising=False)
+ monkeypatch.delenv("OPENAI_MODEL_NAME", raising=False)
+
+ llm = _resolve_llm_instance(None)
+
+ assert llm is not None
+ assert getattr(llm, "model", "") == DEFAULT_LLM_MODEL
+
+
class TestHumanFeedbackResult:
"""Tests for HumanFeedbackResult dataclass."""
@@ -318,6 +366,41 @@ class TestHumanFeedbackExecution:
# But the outcome is still correctly set for routing purposes
assert flow.last_human_feedback.outcome == "approved"
+ @patch("builtins.input", return_value="Approved!")
+ @patch("builtins.print")
+ def test_omitted_llm_collapse_uses_project_model(
+ self, mock_print, mock_input, monkeypatch: pytest.MonkeyPatch
+ ):
+ """When emit is set without llm=, collapse uses MODEL via create_llm."""
+ monkeypatch.setenv("MODEL", "anthropic/claude-sonnet-4-5")
+ monkeypatch.delenv("MODEL_NAME", raising=False)
+ monkeypatch.delenv("OPENAI_MODEL_NAME", raising=False)
+
+ class TestFlow(Flow):
+ @start()
+ @human_feedback(
+ message="Review:",
+ emit=["approved", "rejected"],
+ )
+ def review(self):
+ return "Content"
+
+ flow = TestFlow()
+
+ with (
+ patch.object(
+ flow, "_request_human_feedback", return_value="Looks great, approved!"
+ ),
+ patch.object(flow, "_collapse_to_outcome", return_value="approved") as mock_collapse,
+ ):
+ result = flow.kickoff()
+
+ assert result == "Content"
+ mock_collapse.assert_called_once()
+ collapse_llm = mock_collapse.call_args.kwargs["llm"]
+ assert collapse_llm is not None
+ assert "claude-sonnet-4-5" in getattr(collapse_llm, "model", "")
+
class TestHumanFeedbackHistory:
"""Tests for human feedback history tracking."""
@@ -405,8 +488,8 @@ class TestCollapseToOutcome:
assert result == "approved"
- def test_fallback_to_first(self):
- """Test that unmatched response falls back to first outcome."""
+ def test_unmatched_response_raises(self):
+ """Unmatched LLM text fails closed instead of taking emit[0]."""
flow = Flow()
with patch("crewai.llm.LLM") as MockLLM:
@@ -414,31 +497,28 @@ class TestCollapseToOutcome:
mock_llm.call.return_value = "something completely different"
MockLLM.return_value = mock_llm
- result = flow._collapse_to_outcome(
- feedback="Unclear feedback",
- outcomes=["approved", "rejected"],
- llm="gpt-4o-mini",
- )
-
- assert result == "approved"
+ with pytest.raises(HumanFeedbackCollapseError, match="Could not match"):
+ flow._collapse_to_outcome(
+ feedback="Unclear feedback",
+ outcomes=["approved", "rejected"],
+ llm="gpt-4o-mini",
+ )
- def test_both_llm_calls_fail_returns_first_outcome(self):
- """When both structured and simple prompting fail, return outcomes[0]."""
+ def test_both_llm_calls_fail_raises(self):
+ """When both structured and simple prompting fail, raise instead of emit[0]."""
flow = Flow()
with patch("crewai.llm.LLM") as MockLLM:
mock_llm = MagicMock()
- # Both calls raise — simulates wrong provider / auth failure
mock_llm.call.side_effect = RuntimeError("Model not found")
MockLLM.return_value = mock_llm
- result = flow._collapse_to_outcome(
- feedback="looks great, approve it",
- outcomes=["needs_changes", "approved"],
- llm="gemini-3-flash-preview",
- )
-
- assert result == "needs_changes" # First in list (safe fallback)
+ with pytest.raises(HumanFeedbackCollapseError, match="Could not classify"):
+ flow._collapse_to_outcome(
+ feedback="looks great, approve it",
+ outcomes=["needs_changes", "approved"],
+ llm="gemini-3-flash-preview",
+ )
def test_structured_fails_but_simple_succeeds(self):
"""When structured output fails but simple prompting works, use that."""
@@ -460,7 +540,70 @@ class TestCollapseToOutcome:
assert result == "approved"
+ def test_collapse_failure_does_not_route_to_first_emit(self):
+ """A failed collapse must not fire the first emit listener."""
+ routed: list[str] = []
+
+ class TestFlow(Flow):
+ @start()
+ @human_feedback(
+ message="Review:",
+ emit=["approved", "rejected"],
+ llm="gpt-4o-mini",
+ )
+ def review(self):
+ return "payment of $10"
+
+ @listen("approved")
+ def send_money(self):
+ routed.append("approved")
+ return "sent"
+
+ @listen("rejected")
+ def stop(self):
+ routed.append("rejected")
+ return "stopped"
+ flow = TestFlow()
+ with (
+ patch.object(
+ flow, "_request_human_feedback", return_value="no, reject this"
+ ),
+ patch.object(
+ flow,
+ "_collapse_to_outcome",
+ side_effect=HumanFeedbackCollapseError("failed"),
+ ),
+ ):
+ with pytest.raises(HumanFeedbackCollapseError, match="failed"):
+ flow.kickoff()
+
+ assert routed == []
+
+ def test_unresolved_collapse_llm_raises(self):
+ """If no LLM can be resolved, do not take emit[0]."""
+
+ class TestFlow(Flow):
+ @start()
+ @human_feedback(
+ message="Review:",
+ emit=["approved", "rejected"],
+ )
+ def review(self):
+ return "payment of $10"
+
+ flow = TestFlow()
+ with (
+ patch.object(
+ flow, "_request_human_feedback", return_value="no, reject this"
+ ),
+ patch(
+ "crewai.flow.runtime._resolve_llm_instance",
+ return_value=None,
+ ),
+ ):
+ with pytest.raises(HumanFeedbackCollapseError, match="Could not resolve"):
+ flow.kickoff()
class TestHumanFeedbackLearn:
@@ -596,8 +739,8 @@ class TestHumanFeedbackLearn:
flow.memory.remember_many.assert_not_called()
- def test_learn_true_uses_default_llm(self):
- """When learn=True and llm is not explicitly set, the default gpt-5.4-mini is used."""
+ def test_learn_true_omits_llm_by_default(self):
+ """When learn=True and llm is not set, config.llm stays None for runtime resolve."""
@human_feedback(message="Review:", learn=True)
def test_method(self):
@@ -606,8 +749,7 @@ class TestHumanFeedbackLearn:
config = test_method.__human_feedback_config__
assert config is not None
assert config.learn is True
- # llm defaults to "gpt-5.4-mini" at the function level
- assert config.llm == "gpt-5.4-mini"
+ assert config.llm is None
def test_pre_review_failure_logs_and_returns_raw_output(self, caplog):
"""Pre-review LLM failure falls back to raw output AND logs a warning."""