Platform for stateful agents: AI with advanced memory that can learn and self-improve over time.
feat: add invalid_llm_response stop reason [LET-4083] (#4269)
cthomas3e6a223d17dc
diff --git a/alembic/versions/887a4367b560_convert_stop_reason_from_enum_to_string.py b/alembic/versions/887a4367b560_convert_stop_reason_from_enum_to_string.pynew file mode 100644index 000000000..e3302993b--- /dev/null+++ b/alembic/versions/887a4367b560_convert_stop_reason_from_enum_to_string.py@@ -0,0 +1,39 @@+"""convert_stop_reason_from_enum_to_string++Revision ID: 887a4367b560+Revises: d5103ee17ed5+Create Date: 2025-08-27 16:34:45.605580++"""++from typing import Sequence, Union++from alembic import op+from letta.settings import settings++# revision identifiers, used by Alembic.+revision: str = "887a4367b560"+down_revision: Union[str, None] = "d5103ee17ed5"+branch_labels: Union[str, Sequence[str], None] = None+depends_on: Union[str, Sequence[str], None] = None+++def upgrade() -> None:+ # Skip this migration for SQLite it doesn't enforce column types strictly,+ # so the existing enum values will continue to work as strings.+ if not settings.letta_pg_uri_no_default:+ return++ op.execute(+ """+ ALTER TABLE steps+ ALTER COLUMN stop_reason TYPE VARCHAR+ USING stop_reason::VARCHAR+ """+ )+++def downgrade() -> None:+ # This is a one-way migration as we can't easily recreate the enum type+ # If needed, you would need to create the enum type and cast back+ passdiff --git a/letta/agents/letta_agent.py b/letta/agents/letta_agent.pyindex ea2472e00..6adf85977 100644--- a/letta/agents/letta_agent.py+++ b/letta/agents/letta_agent.py@@ -285,7 +285,11 @@ class LettaAgent(BaseAgent):step_progression = StepProgression.RESPONSE_RECEIVEDlog_event("agent.stream_no_tokens.llm_response.received") # [3^]- response = llm_client.convert_response_to_chat_completion(response_data, in_context_messages, agent_state.llm_config)+ try:+ response = llm_client.convert_response_to_chat_completion(response_data, in_context_messages, agent_state.llm_config)+ except ValueError as e:+ stop_reason = LettaStopReason(stop_reason=StopReasonType.invalid_llm_response.value)+ raise e# update usageusage.step_count += 1@@ -395,8 +399,12 @@ class LettaAgent(BaseAgent):stop_reason = LettaStopReason(stop_reason=StopReasonType.error.value)elif stop_reason.stop_reason in (StopReasonType.end_turn, StopReasonType.max_steps, StopReasonType.tool_rule):self.logger.error("Error occurred during step processing, with valid stop reason: %s", stop_reason.stop_reason)- elif stop_reason.stop_reason not in (StopReasonType.no_tool_call, StopReasonType.invalid_tool_call):- raise ValueError(f"Invalid Stop Reason: {stop_reason}")+ elif stop_reason.stop_reason not in (+ StopReasonType.no_tool_call,+ StopReasonType.invalid_tool_call,+ StopReasonType.invalid_llm_response,+ ):+ self.logger.error("Error occurred during step processing, with unexpected stop reason: %s", stop_reason.stop_reason)# Send error stop reason to client and re-raiseyield f"data: {stop_reason.model_dump_json()}\n\n", 500@@ -582,7 +590,11 @@ class LettaAgent(BaseAgent):step_progression = StepProgression.RESPONSE_RECEIVEDlog_event("agent.step.llm_response.received") # [3^]- response = llm_client.convert_response_to_chat_completion(response_data, in_context_messages, agent_state.llm_config)+ try:+ response = llm_client.convert_response_to_chat_completion(response_data, in_context_messages, agent_state.llm_config)+ except ValueError as e:+ stop_reason = LettaStopReason(stop_reason=StopReasonType.invalid_llm_response.value)+ raise eusage.step_count += 1usage.completion_tokens += response.usage.completion_tokens@@ -683,8 +695,12 @@ class LettaAgent(BaseAgent):stop_reason = LettaStopReason(stop_reason=StopReasonType.error.value)elif stop_reason.stop_reason in (StopReasonType.end_turn, StopReasonType.max_steps, StopReasonType.tool_rule):self.logger.error("Error occurred during step processing, with valid stop reason: %s", stop_reason.stop_reason)- elif stop_reason.stop_reason not in (StopReasonType.no_tool_call, StopReasonType.invalid_tool_call):- raise ValueError(f"Invalid Stop Reason: {stop_reason}")+ elif stop_reason.stop_reason not in (+ StopReasonType.no_tool_call,+ StopReasonType.invalid_tool_call,+ StopReasonType.invalid_llm_response,+ ):+ self.logger.error("Error occurred during step processing, with unexpected stop reason: %s", stop_reason.stop_reason)raise# Update step if it needs to be updated@@ -1076,8 +1092,12 @@ class LettaAgent(BaseAgent):stop_reason = LettaStopReason(stop_reason=StopReasonType.error.value)elif stop_reason.stop_reason in (StopReasonType.end_turn, StopReasonType.max_steps, StopReasonType.tool_rule):self.logger.error("Error occurred during step processing, with valid stop reason: %s", stop_reason.stop_reason)- elif stop_reason.stop_reason not in (StopReasonType.no_tool_call, StopReasonType.invalid_tool_call):- raise ValueError(f"Invalid Stop Reason: {stop_reason}")+ elif stop_reason.stop_reason not in (+ StopReasonType.no_tool_call,+ StopReasonType.invalid_tool_call,+ StopReasonType.invalid_llm_response,+ ):+ self.logger.error("Error occurred during step processing, with unexpected stop reason: %s", stop_reason.stop_reason)# Send error stop reason to client and re-raise with expected response codeyield f"data: {stop_reason.model_dump_json()}\n\n", 500diff --git a/letta/orm/step.py b/letta/orm/step.pyindex a70e831fb..49e90c42f 100644--- a/letta/orm/step.py+++ b/letta/orm/step.py@@ -7,7 +7,6 @@ from sqlalchemy.orm import Mapped, mapped_column, relationshipfrom letta.orm.mixins import ProjectMixinfrom letta.orm.sqlalchemy_base import SqlalchemyBasefrom letta.schemas.enums import StepStatus-from letta.schemas.letta_stop_reason import StopReasonTypefrom letta.schemas.step import Step as PydanticStepif TYPE_CHECKING:@@ -51,7 +50,7 @@ class Step(SqlalchemyBase, ProjectMixin):prompt_tokens: Mapped[int] = mapped_column(default=0, doc="Number of tokens in the prompt")total_tokens: Mapped[int] = mapped_column(default=0, doc="Total number of tokens processed by the agent")completion_tokens_details: Mapped[Optional[Dict]] = mapped_column(JSON, nullable=True, doc="metadata for the agent.")- stop_reason: Mapped[Optional[StopReasonType]] = mapped_column(None, nullable=True, doc="The stop reason associated with this step.")+ stop_reason: Mapped[Optional[str]] = mapped_column(None, nullable=True, doc="The stop reason associated with this step.")tags: Mapped[Optional[List]] = mapped_column(JSON, doc="Metadata tags.")tid: Mapped[Optional[str]] = mapped_column(None, nullable=True, doc="Transaction ID that processed the step.")trace_id: Mapped[Optional[str]] = mapped_column(None, nullable=True, doc="The trace id of the agent step.")diff --git a/letta/schemas/letta_stop_reason.py b/letta/schemas/letta_stop_reason.pyindex 8a0955f72..60365ef40 100644--- a/letta/schemas/letta_stop_reason.py+++ b/letta/schemas/letta_stop_reason.py@@ -9,6 +9,7 @@ from letta.schemas.enums import JobStatusclass StopReasonType(str, Enum):end_turn = "end_turn"error = "error"+ invalid_llm_response = "invalid_llm_response"invalid_tool_call = "invalid_tool_call"max_steps = "max_steps"no_tool_call = "no_tool_call"@@ -23,7 +24,12 @@ class StopReasonType(str, Enum):StopReasonType.tool_rule,):return JobStatus.completed- elif self in (StopReasonType.error, StopReasonType.invalid_tool_call, StopReasonType.no_tool_call):+ elif self in (+ StopReasonType.error,+ StopReasonType.invalid_tool_call,+ StopReasonType.no_tool_call,+ StopReasonType.invalid_llm_response,+ ):return JobStatus.failedelif self == StopReasonType.cancelled:return JobStatus.cancelled