Skip to content

armgit / letta

Public Imported from GitHub
0

Platform for stateful agents: AI with advanced memory that can learn and self-improve over time.

feat: add invalid_llm_response stop reason [LET-4083] (#4269)

cthomas3e6a223d17dc
diff --git a/alembic/versions/887a4367b560_convert_stop_reason_from_enum_to_string.py b/alembic/versions/887a4367b560_convert_stop_reason_from_enum_to_string.py
new file mode 100644
index 000000000..e3302993b
--- /dev/null
+++ b/alembic/versions/887a4367b560_convert_stop_reason_from_enum_to_string.py
@@ -0,0 +1,39 @@
+"""convert_stop_reason_from_enum_to_string
+
+Revision ID: 887a4367b560
+Revises: d5103ee17ed5
+Create Date: 2025-08-27 16:34:45.605580
+
+"""
+
+from typing import Sequence, Union
+
+from alembic import op
+from letta.settings import settings
+
+# revision identifiers, used by Alembic.
+revision: str = "887a4367b560"
+down_revision: Union[str, None] = "d5103ee17ed5"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+ # Skip this migration for SQLite it doesn't enforce column types strictly,
+ # so the existing enum values will continue to work as strings.
+ if not settings.letta_pg_uri_no_default:
+ return
+
+ op.execute(
+ """
+ ALTER TABLE steps
+ ALTER COLUMN stop_reason TYPE VARCHAR
+ USING stop_reason::VARCHAR
+ """
+ )
+
+
+def downgrade() -> None:
+ # This is a one-way migration as we can't easily recreate the enum type
+ # If needed, you would need to create the enum type and cast back
+ pass
diff --git a/letta/agents/letta_agent.py b/letta/agents/letta_agent.py
index ea2472e00..6adf85977 100644
--- a/letta/agents/letta_agent.py
+++ b/letta/agents/letta_agent.py
@@ -285,7 +285,11 @@ class LettaAgent(BaseAgent):
step_progression = StepProgression.RESPONSE_RECEIVED
log_event("agent.stream_no_tokens.llm_response.received") # [3^]
- response = llm_client.convert_response_to_chat_completion(response_data, in_context_messages, agent_state.llm_config)
+ try:
+ response = llm_client.convert_response_to_chat_completion(response_data, in_context_messages, agent_state.llm_config)
+ except ValueError as e:
+ stop_reason = LettaStopReason(stop_reason=StopReasonType.invalid_llm_response.value)
+ raise e
# update usage
usage.step_count += 1
@@ -395,8 +399,12 @@ class LettaAgent(BaseAgent):
stop_reason = LettaStopReason(stop_reason=StopReasonType.error.value)
elif stop_reason.stop_reason in (StopReasonType.end_turn, StopReasonType.max_steps, StopReasonType.tool_rule):
self.logger.error("Error occurred during step processing, with valid stop reason: %s", stop_reason.stop_reason)
- elif stop_reason.stop_reason not in (StopReasonType.no_tool_call, StopReasonType.invalid_tool_call):
- raise ValueError(f"Invalid Stop Reason: {stop_reason}")
+ elif stop_reason.stop_reason not in (
+ StopReasonType.no_tool_call,
+ StopReasonType.invalid_tool_call,
+ StopReasonType.invalid_llm_response,
+ ):
+ self.logger.error("Error occurred during step processing, with unexpected stop reason: %s", stop_reason.stop_reason)
# Send error stop reason to client and re-raise
yield f"data: {stop_reason.model_dump_json()}\n\n", 500
@@ -582,7 +590,11 @@ class LettaAgent(BaseAgent):
step_progression = StepProgression.RESPONSE_RECEIVED
log_event("agent.step.llm_response.received") # [3^]
- response = llm_client.convert_response_to_chat_completion(response_data, in_context_messages, agent_state.llm_config)
+ try:
+ response = llm_client.convert_response_to_chat_completion(response_data, in_context_messages, agent_state.llm_config)
+ except ValueError as e:
+ stop_reason = LettaStopReason(stop_reason=StopReasonType.invalid_llm_response.value)
+ raise e
usage.step_count += 1
usage.completion_tokens += response.usage.completion_tokens
@@ -683,8 +695,12 @@ class LettaAgent(BaseAgent):
stop_reason = LettaStopReason(stop_reason=StopReasonType.error.value)
elif stop_reason.stop_reason in (StopReasonType.end_turn, StopReasonType.max_steps, StopReasonType.tool_rule):
self.logger.error("Error occurred during step processing, with valid stop reason: %s", stop_reason.stop_reason)
- elif stop_reason.stop_reason not in (StopReasonType.no_tool_call, StopReasonType.invalid_tool_call):
- raise ValueError(f"Invalid Stop Reason: {stop_reason}")
+ elif stop_reason.stop_reason not in (
+ StopReasonType.no_tool_call,
+ StopReasonType.invalid_tool_call,
+ StopReasonType.invalid_llm_response,
+ ):
+ self.logger.error("Error occurred during step processing, with unexpected stop reason: %s", stop_reason.stop_reason)
raise
# Update step if it needs to be updated
@@ -1076,8 +1092,12 @@ class LettaAgent(BaseAgent):
stop_reason = LettaStopReason(stop_reason=StopReasonType.error.value)
elif stop_reason.stop_reason in (StopReasonType.end_turn, StopReasonType.max_steps, StopReasonType.tool_rule):
self.logger.error("Error occurred during step processing, with valid stop reason: %s", stop_reason.stop_reason)
- elif stop_reason.stop_reason not in (StopReasonType.no_tool_call, StopReasonType.invalid_tool_call):
- raise ValueError(f"Invalid Stop Reason: {stop_reason}")
+ elif stop_reason.stop_reason not in (
+ StopReasonType.no_tool_call,
+ StopReasonType.invalid_tool_call,
+ StopReasonType.invalid_llm_response,
+ ):
+ self.logger.error("Error occurred during step processing, with unexpected stop reason: %s", stop_reason.stop_reason)
# Send error stop reason to client and re-raise with expected response code
yield f"data: {stop_reason.model_dump_json()}\n\n", 500
diff --git a/letta/orm/step.py b/letta/orm/step.py
index a70e831fb..49e90c42f 100644
--- a/letta/orm/step.py
+++ b/letta/orm/step.py
@@ -7,7 +7,6 @@ from sqlalchemy.orm import Mapped, mapped_column, relationship
from letta.orm.mixins import ProjectMixin
from letta.orm.sqlalchemy_base import SqlalchemyBase
from letta.schemas.enums import StepStatus
-from letta.schemas.letta_stop_reason import StopReasonType
from letta.schemas.step import Step as PydanticStep
if TYPE_CHECKING:
@@ -51,7 +50,7 @@ class Step(SqlalchemyBase, ProjectMixin):
prompt_tokens: Mapped[int] = mapped_column(default=0, doc="Number of tokens in the prompt")
total_tokens: Mapped[int] = mapped_column(default=0, doc="Total number of tokens processed by the agent")
completion_tokens_details: Mapped[Optional[Dict]] = mapped_column(JSON, nullable=True, doc="metadata for the agent.")
- stop_reason: Mapped[Optional[StopReasonType]] = mapped_column(None, nullable=True, doc="The stop reason associated with this step.")
+ stop_reason: Mapped[Optional[str]] = mapped_column(None, nullable=True, doc="The stop reason associated with this step.")
tags: Mapped[Optional[List]] = mapped_column(JSON, doc="Metadata tags.")
tid: Mapped[Optional[str]] = mapped_column(None, nullable=True, doc="Transaction ID that processed the step.")
trace_id: Mapped[Optional[str]] = mapped_column(None, nullable=True, doc="The trace id of the agent step.")
diff --git a/letta/schemas/letta_stop_reason.py b/letta/schemas/letta_stop_reason.py
index 8a0955f72..60365ef40 100644
--- a/letta/schemas/letta_stop_reason.py
+++ b/letta/schemas/letta_stop_reason.py
@@ -9,6 +9,7 @@ from letta.schemas.enums import JobStatus
class StopReasonType(str, Enum):
end_turn = "end_turn"
error = "error"
+ invalid_llm_response = "invalid_llm_response"
invalid_tool_call = "invalid_tool_call"
max_steps = "max_steps"
no_tool_call = "no_tool_call"
@@ -23,7 +24,12 @@ class StopReasonType(str, Enum):
StopReasonType.tool_rule,
):
return JobStatus.completed
- elif self in (StopReasonType.error, StopReasonType.invalid_tool_call, StopReasonType.no_tool_call):
+ elif self in (
+ StopReasonType.error,
+ StopReasonType.invalid_tool_call,
+ StopReasonType.no_tool_call,
+ StopReasonType.invalid_llm_response,
+ ):
return JobStatus.failed
elif self == StopReasonType.cancelled:
return JobStatus.cancelled