Platform for stateful agents: AI with advanced memory that can learn and self-improve over time.
feat: Implement streaming chat completions endpoint [LET-5485] (#5446)
Matthew Zhou2dae4d33c3a2
diff --git a/fern/openapi.json b/fern/openapi.jsonindex bc4b8b74a..8270691ca 100644--- a/fern/openapi.json+++ b/fern/openapi.json@@ -6726,6 +6726,50 @@}}},+ "/v1/chat/completions": {+ "post": {+ "tags": ["chat"],+ "summary": "Create Chat Completion",+ "description": "Create a chat completion using a Letta agent (OpenAI-compatible).\n\nThis endpoint provides full OpenAI API compatibility. The agent is selected based on:\n- The 'model' parameter in the request (should contain an agent ID in format 'agent-...')\n\nWhen streaming is enabled (stream=true), the response will be Server-Sent Events\nwith ChatCompletionChunk objects.",+ "operationId": "create_chat_completion",+ "parameters": [],+ "requestBody": {+ "required": true,+ "content": {+ "application/json": {+ "schema": {+ "$ref": "#/components/schemas/ChatCompletionRequest"+ }+ }+ }+ },+ "responses": {+ "200": {+ "description": "Successful response",+ "content": {+ "application/json": {+ "schema": {+ "$ref": "#/components/schemas/ChatCompletion"+ }+ },+ "text/event-stream": {+ "description": "Server-Sent Events stream (when stream=true)"+ }+ }+ },+ "422": {+ "description": "Validation Error",+ "content": {+ "application/json": {+ "schema": {+ "$ref": "#/components/schemas/HTTPValidationError"+ }+ }+ }+ }+ }+ }+ },"/v1/groups/": {"get": {"tags": ["groups"],@@ -22292,6 +22336,46 @@"title": "AgentType","description": "Enum to represent the type of agent."},+ "Annotation": {+ "properties": {+ "type": {+ "type": "string",+ "const": "url_citation",+ "title": "Type"+ },+ "url_citation": {+ "$ref": "#/components/schemas/AnnotationURLCitation"+ }+ },+ "additionalProperties": true,+ "type": "object",+ "required": ["type", "url_citation"],+ "title": "Annotation"+ },+ "AnnotationURLCitation": {+ "properties": {+ "end_index": {+ "type": "integer",+ "title": "End Index"+ },+ "start_index": {+ "type": "integer",+ "title": "Start Index"+ },+ "title": {+ "type": "string",+ "title": "Title"+ },+ "url": {+ "type": "string",+ "title": "Url"+ }+ },+ "additionalProperties": true,+ "type": "object",+ "required": ["end_index", "start_index", "title", "url"],+ "title": "AnnotationURLCitation"+ },"ApprovalCreate": {"properties": {"type": {@@ -22999,6 +23083,17 @@"title": "AssistantMessage","description": "A message sent by the LLM in response to user input. Used in the LLM context.\n\nArgs:\n id (str): The ID of the message\n date (datetime): The date the message was created in ISO format\n name (Optional[str]): The name of the sender of the message\n content (Union[str, List[LettaAssistantMessageContentUnion]]): The message content sent by the agent (can be a string or an array of content parts)"},+ "Audio": {+ "properties": {+ "id": {+ "type": "string",+ "title": "Id"+ }+ },+ "type": "object",+ "required": ["id"],+ "title": "Audio"+ },"AuthRequest": {"properties": {"password": {@@ -23925,197 +24020,127 @@"type": "object","title": "CancelAgentRunRequest"},- "ChatCompletionMessageFunctionToolCall": {+ "ChatCompletion": {"properties": {"id": {"type": "string","title": "Id"},- "function": {- "$ref": "#/components/schemas/Function"+ "choices": {+ "items": {+ "$ref": "#/components/schemas/Choice"+ },+ "type": "array",+ "title": "Choices"},- "type": {- "type": "string",- "const": "function",- "title": "Type"- }- },- "additionalProperties": true,- "type": "object",- "required": ["id", "function", "type"],- "title": "ChatCompletionMessageFunctionToolCall"- },- "ChildToolRule": {- "properties": {- "tool_name": {+ "created": {+ "type": "integer",+ "title": "Created"+ },+ "model": {"type": "string",- "title": "Tool Name",- "description": "The name of the tool. Must exist in the database for the user's organization."+ "title": "Model"},- "type": {+ "object": {"type": "string",- "const": "constrain_child_tools",- "title": "Type",- "default": "constrain_child_tools"+ "const": "chat.completion",+ "title": "Object"},- "prompt_template": {+ "service_tier": {"anyOf": [{- "type": "string"+ "type": "string",+ "enum": ["auto", "default", "flex", "scale", "priority"]},{"type": "null"}],- "title": "Prompt Template",- "description": "Optional template string (ignored)."- },- "children": {- "items": {- "type": "string"- },- "type": "array",- "title": "Children",- "description": "The children tools that can be invoked."+ "title": "Service Tier"},- "child_arg_nodes": {+ "system_fingerprint": {"anyOf": [{- "items": {- "$ref": "#/components/schemas/ToolCallNode"- },- "type": "array"+ "type": "string"},{"type": "null"}],- "title": "Child Arg Nodes",- "description": "Optional list of typed child argument overrides. Each node must reference a child in 'children'."- }- },- "additionalProperties": false,- "type": "object",- "required": ["tool_name", "children"],- "title": "ChildToolRule",- "description": "A ToolRule represents a tool that can be invoked by the agent."- },- "ChildToolRuleSchema": {- "properties": {- "tool_name": {- "type": "string",- "title": "Tool Name"- },- "type": {- "type": "string",- "title": "Type"- },- "children": {- "items": {- "type": "string"- },- "type": "array",- "title": "Children"- }- },- "type": "object",- "required": ["tool_name", "type", "children"],- "title": "ChildToolRuleSchema"- },- "CodeInput": {- "properties": {- "code": {- "type": "string",- "title": "Code",- "description": "Source code to parse for JSON schema"+ "title": "System Fingerprint"},- "source_type": {+ "usage": {"anyOf": [{- "type": "string"+ "$ref": "#/components/schemas/CompletionUsage"},{"type": "null"}- ],- "title": "Source Type",- "description": "The source type of the code (python or typescript)",- "default": "python"+ ]}},+ "additionalProperties": true,"type": "object",- "required": ["code"],- "title": "CodeInput"+ "required": ["id", "choices", "created", "model", "object"],+ "title": "ChatCompletion"},- "ConditionalToolRule": {+ "ChatCompletionAssistantMessageParam": {"properties": {- "tool_name": {- "type": "string",- "title": "Tool Name",- "description": "The name of the tool. Must exist in the database for the user's organization."- },- "type": {+ "role": {"type": "string",- "const": "conditional",- "title": "Type",- "default": "conditional"+ "const": "assistant",+ "title": "Role"},- "prompt_template": {+ "audio": {"anyOf": [{- "type": "string"+ "$ref": "#/components/schemas/Audio"},{"type": "null"}- ],- "title": "Prompt Template",- "description": "Optional template string (ignored)."+ ]},- "default_child": {+ "content": {"anyOf": [{"type": "string"},+ {+ "items": {+ "anyOf": [+ {+ "$ref": "#/components/schemas/ChatCompletionContentPartTextParam"+ },+ {+ "$ref": "#/components/schemas/ChatCompletionContentPartRefusalParam"+ }+ ]+ },+ "type": "array"+ },{"type": "null"}],- "title": "Default Child",- "description": "The default child tool to be called. If None, any tool can be called."- },- "child_output_mapping": {- "additionalProperties": {- "type": "string"- },- "type": "object",- "title": "Child Output Mapping",- "description": "The output case to check for mapping"+ "title": "Content"},- "require_output_mapping": {- "type": "boolean",- "title": "Require Output Mapping",- "description": "Whether to throw an error when output doesn't match any case",- "default": false- }- },- "additionalProperties": false,- "type": "object",- "required": ["tool_name", "child_output_mapping"],- "title": "ConditionalToolRule",- "description": "A ToolRule that conditionally maps to different child tools based on the output."- },- "ConditionalToolRuleSchema": {- "properties": {- "tool_name": {- "type": "string",- "title": "Tool Name"+ "function_call": {+ "anyOf": [+ {+ "$ref": "#/components/schemas/FunctionCall-Input"+ },+ {+ "type": "null"+ }+ ]},- "type": {+ "name": {"type": "string",- "title": "Type"+ "title": "Name"},- "default_child": {+ "refusal": {"anyOf": [{"type": "string"@@ -24124,17 +24149,1015 @@"type": "null"}],- "title": "Default Child"+ "title": "Refusal"},- "child_output_mapping": {- "additionalProperties": {- "type": "string"+ "tool_calls": {+ "items": {+ "anyOf": [+ {+ "$ref": "#/components/schemas/ChatCompletionMessageFunctionToolCallParam"+ },+ {+ "$ref": "#/components/schemas/ChatCompletionMessageCustomToolCallParam"+ }+ ]},- "type": "object",- "title": "Child Output Mapping"- },- "require_output_mapping": {- "type": "boolean",+ "type": "array",+ "title": "Tool Calls"+ }+ },+ "type": "object",+ "required": ["role"],+ "title": "ChatCompletionAssistantMessageParam"+ },+ "ChatCompletionAudio": {+ "properties": {+ "id": {+ "type": "string",+ "title": "Id"+ },+ "data": {+ "type": "string",+ "title": "Data"+ },+ "expires_at": {+ "type": "integer",+ "title": "Expires At"+ },+ "transcript": {+ "type": "string",+ "title": "Transcript"+ }+ },+ "additionalProperties": true,+ "type": "object",+ "required": ["id", "data", "expires_at", "transcript"],+ "title": "ChatCompletionAudio"+ },+ "ChatCompletionContentPartImageParam": {+ "properties": {+ "image_url": {+ "$ref": "#/components/schemas/ImageURL"+ },+ "type": {+ "type": "string",+ "const": "image_url",+ "title": "Type"+ }+ },+ "type": "object",+ "required": ["image_url", "type"],+ "title": "ChatCompletionContentPartImageParam"+ },+ "ChatCompletionContentPartInputAudioParam": {+ "properties": {+ "input_audio": {+ "$ref": "#/components/schemas/InputAudio"+ },+ "type": {+ "type": "string",+ "const": "input_audio",+ "title": "Type"+ }+ },+ "type": "object",+ "required": ["input_audio", "type"],+ "title": "ChatCompletionContentPartInputAudioParam"+ },+ "ChatCompletionContentPartRefusalParam": {+ "properties": {+ "refusal": {+ "type": "string",+ "title": "Refusal"+ },+ "type": {+ "type": "string",+ "const": "refusal",+ "title": "Type"+ }+ },+ "type": "object",+ "required": ["refusal", "type"],+ "title": "ChatCompletionContentPartRefusalParam"+ },+ "ChatCompletionContentPartTextParam": {+ "properties": {+ "text": {+ "type": "string",+ "title": "Text"+ },+ "type": {+ "type": "string",+ "const": "text",+ "title": "Type"+ }+ },+ "type": "object",+ "required": ["text", "type"],+ "title": "ChatCompletionContentPartTextParam"+ },+ "ChatCompletionDeveloperMessageParam": {+ "properties": {+ "content": {+ "anyOf": [+ {+ "type": "string"+ },+ {+ "items": {+ "$ref": "#/components/schemas/ChatCompletionContentPartTextParam"+ },+ "type": "array"+ }+ ],+ "title": "Content"+ },+ "role": {+ "type": "string",+ "const": "developer",+ "title": "Role"+ },+ "name": {+ "type": "string",+ "title": "Name"+ }+ },+ "type": "object",+ "required": ["content", "role"],+ "title": "ChatCompletionDeveloperMessageParam"+ },+ "ChatCompletionFunctionMessageParam": {+ "properties": {+ "content": {+ "anyOf": [+ {+ "type": "string"+ },+ {+ "type": "null"+ }+ ],+ "title": "Content"+ },+ "name": {+ "type": "string",+ "title": "Name"+ },+ "role": {+ "type": "string",+ "const": "function",+ "title": "Role"+ }+ },+ "type": "object",+ "required": ["content", "name", "role"],+ "title": "ChatCompletionFunctionMessageParam"+ },+ "ChatCompletionMessage": {+ "properties": {+ "content": {+ "anyOf": [+ {+ "type": "string"+ },+ {+ "type": "null"+ }+ ],+ "title": "Content"+ },+ "refusal": {+ "anyOf": [+ {+ "type": "string"+ },+ {+ "type": "null"+ }+ ],+ "title": "Refusal"+ },+ "role": {+ "type": "string",+ "const": "assistant",+ "title": "Role"+ },+ "annotations": {+ "anyOf": [+ {+ "items": {+ "$ref": "#/components/schemas/Annotation"+ },+ "type": "array"+ },+ {+ "type": "null"+ }+ ],+ "title": "Annotations"+ },+ "audio": {+ "anyOf": [+ {+ "$ref": "#/components/schemas/ChatCompletionAudio"+ },+ {+ "type": "null"+ }+ ]+ },+ "function_call": {+ "anyOf": [+ {+ "$ref": "#/components/schemas/FunctionCall-Output"+ },+ {+ "type": "null"+ }+ ]+ },+ "tool_calls": {+ "anyOf": [+ {+ "items": {+ "anyOf": [+ {+ "$ref": "#/components/schemas/ChatCompletionMessageFunctionToolCall-Output"+ },+ {+ "$ref": "#/components/schemas/ChatCompletionMessageCustomToolCall"+ }+ ]+ },+ "type": "array"+ },+ {+ "type": "null"+ }+ ],+ "title": "Tool Calls"+ }+ },+ "additionalProperties": true,+ "type": "object",+ "required": ["role"],+ "title": "ChatCompletionMessage"+ },+ "ChatCompletionMessageCustomToolCall": {+ "properties": {+ "id": {+ "type": "string",+ "title": "Id"+ },+ "custom": {+ "$ref": "#/components/schemas/Custom-Output"+ },+ "type": {+ "type": "string",+ "const": "custom",+ "title": "Type"+ }+ },+ "additionalProperties": true,+ "type": "object",+ "required": ["id", "custom", "type"],+ "title": "ChatCompletionMessageCustomToolCall"+ },+ "ChatCompletionMessageCustomToolCallParam": {+ "properties": {+ "id": {+ "type": "string",+ "title": "Id"+ },+ "custom": {+ "$ref": "#/components/schemas/Custom-Input"+ },+ "type": {+ "type": "string",+ "const": "custom",+ "title": "Type"+ }+ },+ "type": "object",+ "required": ["id", "custom", "type"],+ "title": "ChatCompletionMessageCustomToolCallParam"+ },+ "ChatCompletionMessageFunctionToolCall-Input": {+ "properties": {+ "id": {+ "type": "string",+ "title": "Id"+ },+ "function": {+ "$ref": "#/components/schemas/openai__types__chat__chat_completion_message_function_tool_call__Function"+ },+ "type": {+ "type": "string",+ "const": "function",+ "title": "Type"+ }+ },+ "additionalProperties": true,+ "type": "object",+ "required": ["id", "function", "type"],+ "title": "ChatCompletionMessageFunctionToolCall"+ },+ "ChatCompletionMessageFunctionToolCall-Output": {+ "properties": {+ "id": {+ "type": "string",+ "title": "Id"+ },+ "function": {+ "$ref": "#/components/schemas/Function-Output"+ },+ "type": {+ "type": "string",+ "const": "function",+ "title": "Type"+ }+ },+ "additionalProperties": true,+ "type": "object",+ "required": ["id", "function", "type"],+ "title": "ChatCompletionMessageFunctionToolCall"+ },+ "ChatCompletionMessageFunctionToolCallParam": {+ "properties": {+ "id": {+ "type": "string",+ "title": "Id"+ },+ "function": {+ "$ref": "#/components/schemas/openai__types__chat__chat_completion_message_function_tool_call_param__Function"+ },+ "type": {+ "type": "string",+ "const": "function",+ "title": "Type"+ }+ },+ "type": "object",+ "required": ["id", "function", "type"],+ "title": "ChatCompletionMessageFunctionToolCallParam"+ },+ "ChatCompletionRequest": {+ "properties": {+ "model": {+ "type": "string",+ "title": "Model",+ "description": "ID of the model to use"+ },+ "messages": {+ "items": {+ "anyOf": [+ {+ "$ref": "#/components/schemas/ChatCompletionDeveloperMessageParam"+ },+ {+ "$ref": "#/components/schemas/ChatCompletionSystemMessageParam"+ },+ {+ "$ref": "#/components/schemas/ChatCompletionUserMessageParam"+ },+ {+ "$ref": "#/components/schemas/ChatCompletionAssistantMessageParam"+ },+ {+ "$ref": "#/components/schemas/ChatCompletionToolMessageParam"+ },+ {+ "$ref": "#/components/schemas/ChatCompletionFunctionMessageParam"+ }+ ]+ },+ "type": "array",+ "title": "Messages",+ "description": "Messages comprising the conversation so far"+ },+ "temperature": {+ "anyOf": [+ {+ "type": "number",+ "maximum": 2,+ "minimum": 0+ },+ {+ "type": "null"+ }+ ],+ "title": "Temperature",+ "description": "Sampling temperature"+ },+ "top_p": {+ "anyOf": [+ {+ "type": "number",+ "maximum": 1,+ "minimum": 0+ },+ {+ "type": "null"+ }+ ],+ "title": "Top P",+ "description": "Nucleus sampling parameter"+ },+ "n": {+ "anyOf": [+ {+ "type": "integer",+ "minimum": 1+ },+ {+ "type": "null"+ }+ ],+ "title": "N",+ "description": "Number of chat completion choices to generate",+ "default": 1+ },+ "stream": {+ "anyOf": [+ {+ "type": "boolean"+ },+ {+ "type": "null"+ }+ ],+ "title": "Stream",+ "description": "Whether to stream back partial progress",+ "default": false+ },+ "stop": {+ "anyOf": [+ {+ "type": "string"+ },+ {+ "items": {+ "type": "string"+ },+ "type": "array"+ },+ {+ "type": "null"+ }+ ],+ "title": "Stop",+ "description": "Sequences where the API will stop generating"+ },+ "max_tokens": {+ "anyOf": [+ {+ "type": "integer"+ },+ {+ "type": "null"+ }+ ],+ "title": "Max Tokens",+ "description": "Maximum number of tokens to generate"+ },+ "presence_penalty": {+ "anyOf": [+ {+ "type": "number",+ "maximum": 2,+ "minimum": -2+ },+ {+ "type": "null"+ }+ ],+ "title": "Presence Penalty",+ "description": "Presence penalty"+ },+ "frequency_penalty": {+ "anyOf": [+ {+ "type": "number",+ "maximum": 2,+ "minimum": -2+ },+ {+ "type": "null"+ }+ ],+ "title": "Frequency Penalty",+ "description": "Frequency penalty"+ },+ "user": {+ "anyOf": [+ {+ "type": "string"+ },+ {+ "type": "null"+ }+ ],+ "title": "User",+ "description": "A unique identifier representing your end-user"+ }+ },+ "type": "object",+ "required": ["model", "messages"],+ "title": "ChatCompletionRequest",+ "description": "OpenAI-compatible chat completion request - exactly matching OpenAI's schema."+ },+ "ChatCompletionSystemMessageParam": {+ "properties": {+ "content": {+ "anyOf": [+ {+ "type": "string"+ },+ {+ "items": {+ "$ref": "#/components/schemas/ChatCompletionContentPartTextParam"+ },+ "type": "array"+ }+ ],+ "title": "Content"+ },+ "role": {+ "type": "string",+ "const": "system",+ "title": "Role"+ },+ "name": {+ "type": "string",+ "title": "Name"+ }+ },+ "type": "object",+ "required": ["content", "role"],+ "title": "ChatCompletionSystemMessageParam"+ },+ "ChatCompletionTokenLogprob": {+ "properties": {+ "token": {+ "type": "string",+ "title": "Token"+ },+ "bytes": {+ "anyOf": [+ {+ "items": {+ "type": "integer"+ },+ "type": "array"+ },+ {+ "type": "null"+ }+ ],+ "title": "Bytes"+ },+ "logprob": {+ "type": "number",+ "title": "Logprob"+ },+ "top_logprobs": {+ "items": {+ "$ref": "#/components/schemas/TopLogprob"+ },+ "type": "array",+ "title": "Top Logprobs"+ }+ },+ "additionalProperties": true,+ "type": "object",+ "required": ["token", "logprob", "top_logprobs"],+ "title": "ChatCompletionTokenLogprob"+ },+ "ChatCompletionToolMessageParam": {+ "properties": {+ "content": {+ "anyOf": [+ {+ "type": "string"+ },+ {+ "items": {+ "$ref": "#/components/schemas/ChatCompletionContentPartTextParam"+ },+ "type": "array"+ }+ ],+ "title": "Content"+ },+ "role": {+ "type": "string",+ "const": "tool",+ "title": "Role"+ },+ "tool_call_id": {+ "type": "string",+ "title": "Tool Call Id"+ }+ },+ "type": "object",+ "required": ["content", "role", "tool_call_id"],+ "title": "ChatCompletionToolMessageParam"+ },+ "ChatCompletionUserMessageParam": {+ "properties": {+ "content": {+ "anyOf": [+ {+ "type": "string"+ },+ {+ "items": {+ "anyOf": [+ {+ "$ref": "#/components/schemas/ChatCompletionContentPartTextParam"+ },+ {+ "$ref": "#/components/schemas/ChatCompletionContentPartImageParam"+ },+ {+ "$ref": "#/components/schemas/ChatCompletionContentPartInputAudioParam"+ },+ {+ "$ref": "#/components/schemas/File"+ }+ ]+ },+ "type": "array"+ }+ ],+ "title": "Content"+ },+ "role": {+ "type": "string",+ "const": "user",+ "title": "Role"+ },+ "name": {+ "type": "string",+ "title": "Name"+ }+ },+ "type": "object",+ "required": ["content", "role"],+ "title": "ChatCompletionUserMessageParam"+ },+ "ChildToolRule": {+ "properties": {+ "tool_name": {+ "type": "string",+ "title": "Tool Name",+ "description": "The name of the tool. Must exist in the database for the user's organization."+ },+ "type": {+ "type": "string",+ "const": "constrain_child_tools",+ "title": "Type",+ "default": "constrain_child_tools"+ },+ "prompt_template": {+ "anyOf": [+ {+ "type": "string"+ },+ {+ "type": "null"+ }+ ],+ "title": "Prompt Template",+ "description": "Optional template string (ignored)."+ },+ "children": {+ "items": {+ "type": "string"+ },+ "type": "array",+ "title": "Children",+ "description": "The children tools that can be invoked."+ },+ "child_arg_nodes": {+ "anyOf": [+ {+ "items": {+ "$ref": "#/components/schemas/ToolCallNode"+ },+ "type": "array"+ },+ {+ "type": "null"+ }+ ],+ "title": "Child Arg Nodes",+ "description": "Optional list of typed child argument overrides. Each node must reference a child in 'children'."+ }+ },+ "additionalProperties": false,+ "type": "object",+ "required": ["tool_name", "children"],+ "title": "ChildToolRule",+ "description": "A ToolRule represents a tool that can be invoked by the agent."+ },+ "ChildToolRuleSchema": {+ "properties": {+ "tool_name": {+ "type": "string",+ "title": "Tool Name"+ },+ "type": {+ "type": "string",+ "title": "Type"+ },+ "children": {+ "items": {+ "type": "string"+ },+ "type": "array",+ "title": "Children"+ }+ },+ "type": "object",+ "required": ["tool_name", "type", "children"],+ "title": "ChildToolRuleSchema"+ },+ "Choice": {+ "properties": {+ "finish_reason": {+ "type": "string",+ "enum": [+ "stop",+ "length",+ "tool_calls",+ "content_filter",+ "function_call"+ ],+ "title": "Finish Reason"+ },+ "index": {+ "type": "integer",+ "title": "Index"+ },+ "logprobs": {+ "anyOf": [+ {+ "$ref": "#/components/schemas/ChoiceLogprobs"+ },+ {+ "type": "null"+ }+ ]+ },+ "message": {+ "$ref": "#/components/schemas/ChatCompletionMessage"+ }+ },+ "additionalProperties": true,+ "type": "object",+ "required": ["finish_reason", "index", "message"],+ "title": "Choice"+ },+ "ChoiceLogprobs": {+ "properties": {+ "content": {+ "anyOf": [+ {+ "items": {+ "$ref": "#/components/schemas/ChatCompletionTokenLogprob"+ },+ "type": "array"+ },+ {+ "type": "null"+ }+ ],+ "title": "Content"+ },+ "refusal": {+ "anyOf": [+ {+ "items": {+ "$ref": "#/components/schemas/ChatCompletionTokenLogprob"+ },+ "type": "array"+ },+ {+ "type": "null"+ }+ ],+ "title": "Refusal"+ }+ },+ "additionalProperties": true,+ "type": "object",+ "title": "ChoiceLogprobs"+ },+ "CodeInput": {+ "properties": {+ "code": {+ "type": "string",+ "title": "Code",+ "description": "Source code to parse for JSON schema"+ },+ "source_type": {+ "anyOf": [+ {+ "type": "string"+ },+ {+ "type": "null"+ }+ ],+ "title": "Source Type",+ "description": "The source type of the code (python or typescript)",+ "default": "python"+ }+ },+ "type": "object",+ "required": ["code"],+ "title": "CodeInput"+ },+ "CompletionTokensDetails": {+ "properties": {+ "accepted_prediction_tokens": {+ "anyOf": [+ {+ "type": "integer"+ },+ {+ "type": "null"+ }+ ],+ "title": "Accepted Prediction Tokens"+ },+ "audio_tokens": {+ "anyOf": [+ {+ "type": "integer"+ },+ {+ "type": "null"+ }+ ],+ "title": "Audio Tokens"+ },+ "reasoning_tokens": {+ "anyOf": [+ {+ "type": "integer"+ },+ {+ "type": "null"+ }+ ],+ "title": "Reasoning Tokens"+ },+ "rejected_prediction_tokens": {+ "anyOf": [+ {+ "type": "integer"+ },+ {+ "type": "null"+ }+ ],+ "title": "Rejected Prediction Tokens"+ }+ },+ "additionalProperties": true,+ "type": "object",+ "title": "CompletionTokensDetails"+ },+ "CompletionUsage": {+ "properties": {+ "completion_tokens": {+ "type": "integer",+ "title": "Completion Tokens"+ },+ "prompt_tokens": {+ "type": "integer",+ "title": "Prompt Tokens"+ },+ "total_tokens": {+ "type": "integer",+ "title": "Total Tokens"+ },+ "completion_tokens_details": {+ "anyOf": [+ {+ "$ref": "#/components/schemas/CompletionTokensDetails"+ },+ {+ "type": "null"+ }+ ]+ },+ "prompt_tokens_details": {+ "anyOf": [+ {+ "$ref": "#/components/schemas/PromptTokensDetails"+ },+ {+ "type": "null"+ }+ ]+ }+ },+ "additionalProperties": true,+ "type": "object",+ "required": ["completion_tokens", "prompt_tokens", "total_tokens"],+ "title": "CompletionUsage"+ },+ "ConditionalToolRule": {+ "properties": {+ "tool_name": {+ "type": "string",+ "title": "Tool Name",+ "description": "The name of the tool. Must exist in the database for the user's organization."+ },+ "type": {+ "type": "string",+ "const": "conditional",+ "title": "Type",+ "default": "conditional"+ },+ "prompt_template": {+ "anyOf": [+ {+ "type": "string"+ },+ {+ "type": "null"+ }+ ],+ "title": "Prompt Template",+ "description": "Optional template string (ignored)."+ },+ "default_child": {+ "anyOf": [+ {+ "type": "string"+ },+ {+ "type": "null"+ }+ ],+ "title": "Default Child",+ "description": "The default child tool to be called. If None, any tool can be called."+ },+ "child_output_mapping": {+ "additionalProperties": {+ "type": "string"+ },+ "type": "object",+ "title": "Child Output Mapping",+ "description": "The output case to check for mapping"+ },+ "require_output_mapping": {+ "type": "boolean",+ "title": "Require Output Mapping",+ "description": "Whether to throw an error when output doesn't match any case",+ "default": false+ }+ },+ "additionalProperties": false,+ "type": "object",+ "required": ["tool_name", "child_output_mapping"],+ "title": "ConditionalToolRule",+ "description": "A ToolRule that conditionally maps to different child tools based on the output."+ },+ "ConditionalToolRuleSchema": {+ "properties": {+ "tool_name": {+ "type": "string",+ "title": "Tool Name"+ },+ "type": {+ "type": "string",+ "title": "Type"+ },+ "default_child": {+ "anyOf": [+ {+ "type": "string"+ },+ {+ "type": "null"+ }+ ],+ "title": "Default Child"+ },+ "child_output_mapping": {+ "additionalProperties": {+ "type": "string"+ },+ "type": "object",+ "title": "Child Output Mapping"+ },+ "require_output_mapping": {+ "type": "boolean","title": "Require Output Mapping"}},@@ -25188,6 +26211,37 @@"title": "CreateBlock","description": "Create a block"},+ "Custom-Input": {+ "properties": {+ "input": {+ "type": "string",+ "title": "Input"+ },+ "name": {+ "type": "string",+ "title": "Name"+ }+ },+ "type": "object",+ "required": ["input", "name"],+ "title": "Custom"+ },+ "Custom-Output": {+ "properties": {+ "input": {+ "type": "string",+ "title": "Input"+ },+ "name": {+ "type": "string",+ "title": "Name"+ }+ },+ "additionalProperties": true,+ "type": "object",+ "required": ["input", "name"],+ "title": "Custom"+ },"DeleteDeploymentResponse": {"properties": {"deleted_blocks": {@@ -25554,6 +26608,21 @@"enum": ["positive", "negative"],"title": "FeedbackType"},+ "File": {+ "properties": {+ "file": {+ "$ref": "#/components/schemas/FileFile"+ },+ "type": {+ "type": "string",+ "const": "file",+ "title": "Type"+ }+ },+ "type": "object",+ "required": ["file", "type"],+ "title": "File"+ },"FileAgentSchema": {"properties": {"agent_id": {@@ -25855,6 +26924,24 @@"required": ["value", "file_id", "source_id", "is_open"],"title": "FileBlock"},+ "FileFile": {+ "properties": {+ "file_data": {+ "type": "string",+ "title": "File Data"+ },+ "file_id": {+ "type": "string",+ "title": "File Id"+ },+ "filename": {+ "type": "string",+ "title": "Filename"+ }+ },+ "type": "object",+ "title": "FileFile"+ },"FileMetadata": {"properties": {"source_id": {@@ -26343,7 +27430,7 @@"title": "Folder","description": "Representation of a folder, which is a collection of files and passages.\n\nParameters:\n id (str): The ID of the folder\n name (str): The name of the folder.\n embedding_config (EmbeddingConfig): The embedding configuration used by the folder.\n user_id (str): The ID of the user that created the folder.\n metadata (dict): Metadata associated with the folder.\n description (str): The description of the folder."},- "Function": {+ "Function-Output": {"properties": {"arguments": {"type": "string",@@ -26359,6 +27446,37 @@"required": ["arguments", "name"],"title": "Function"},+ "FunctionCall-Input": {+ "properties": {+ "arguments": {+ "type": "string",+ "title": "Arguments"+ },+ "name": {+ "type": "string",+ "title": "Name"+ }+ },+ "type": "object",+ "required": ["arguments", "name"],+ "title": "FunctionCall"+ },+ "FunctionCall-Output": {+ "properties": {+ "arguments": {+ "type": "string",+ "title": "Arguments"+ },+ "name": {+ "type": "string",+ "title": "Name"+ }+ },+ "additionalProperties": true,+ "type": "object",+ "required": ["arguments", "name"],+ "title": "FunctionCall"+ },"FunctionDefinition": {"properties": {"name": {@@ -27516,6 +28634,22 @@"required": ["source"],"title": "ImageContent"},+ "ImageURL": {+ "properties": {+ "url": {+ "type": "string",+ "title": "Url"+ },+ "detail": {+ "type": "string",+ "enum": ["auto", "low", "high"],+ "title": "Detail"+ }+ },+ "type": "object",+ "required": ["url"],+ "title": "ImageURL"+ },"ImportedAgentsResponse": {"properties": {"agent_ids": {@@ -27577,6 +28711,22 @@"title": "InitToolRule","description": "Represents the initial tool rule configuration."},+ "InputAudio": {+ "properties": {+ "data": {+ "type": "string",+ "title": "Data"+ },+ "format": {+ "type": "string",+ "enum": ["wav", "mp3"],+ "title": "Format"+ }+ },+ "type": "object",+ "required": ["data", "format"],+ "title": "InputAudio"+ },"InternalTemplateAgentCreate": {"properties": {"name": {@@ -29922,7 +31072,7 @@"anyOf": [{"items": {- "$ref": "#/components/schemas/ChatCompletionMessageFunctionToolCall"+ "$ref": "#/components/schemas/ChatCompletionMessageFunctionToolCall-Output"},"type": "array"},@@ -30972,6 +32122,35 @@"required": ["name"],"title": "PipRequirement"},+ "PromptTokensDetails": {+ "properties": {+ "audio_tokens": {+ "anyOf": [+ {+ "type": "integer"+ },+ {+ "type": "null"+ }+ ],+ "title": "Audio Tokens"+ },+ "cached_tokens": {+ "anyOf": [+ {+ "type": "integer"+ },+ {+ "type": "null"+ }+ ],+ "title": "Cached Tokens"+ }+ },+ "additionalProperties": true,+ "type": "object",+ "title": "PromptTokensDetails"+ },"Provider": {"properties": {"id": {@@ -34873,6 +36052,36 @@"type": "object","title": "ToolUpdate"},+ "TopLogprob": {+ "properties": {+ "token": {+ "type": "string",+ "title": "Token"+ },+ "bytes": {+ "anyOf": [+ {+ "items": {+ "type": "integer"+ },+ "type": "array"+ },+ {+ "type": "null"+ }+ ],+ "title": "Bytes"+ },+ "logprob": {+ "type": "number",+ "title": "Logprob"+ }+ },+ "additionalProperties": true,+ "type": "object",+ "required": ["token", "logprob"],+ "title": "TopLogprob"+ },"UpdateAgent": {"properties": {"name": {@@ -36712,7 +37921,7 @@"anyOf": [{"items": {- "$ref": "#/components/schemas/ChatCompletionMessageFunctionToolCall"+ "$ref": "#/components/schemas/ChatCompletionMessageFunctionToolCall-Input"},"type": "array"},@@ -37461,6 +38670,37 @@],"title": "ToolSchema"},+ "openai__types__chat__chat_completion_message_function_tool_call__Function": {+ "properties": {+ "arguments": {+ "type": "string",+ "title": "Arguments"+ },+ "name": {+ "type": "string",+ "title": "Name"+ }+ },+ "additionalProperties": true,+ "type": "object",+ "required": ["arguments", "name"],+ "title": "Function"+ },+ "openai__types__chat__chat_completion_message_function_tool_call_param__Function": {+ "properties": {+ "arguments": {+ "type": "string",+ "title": "Arguments"+ },+ "name": {+ "type": "string",+ "title": "Name"+ }+ },+ "type": "object",+ "required": ["arguments", "name"],+ "title": "Function"+ },"LettaMessageUnion": {"oneOf": [{diff --git a/letta/errors.py b/letta/errors.pyindex 5d56e51a1..34a82ccb8 100644--- a/letta/errors.py+++ b/letta/errors.py@@ -142,6 +142,14 @@ class LettaMCPTimeoutError(LettaMCPError):super().__init__(message=message, code=ErrorCode.TIMEOUT, details=details)+class LettaServiceUnavailableError(LettaError):+ """Error raised when a required service is unavailable."""++ def __init__(self, message: str, service_name: Optional[str] = None):+ details = {"service_name": service_name} if service_name else {}+ super().__init__(message=message, code=ErrorCode.INTERNAL_SERVER_ERROR, details=details)++class LettaUnexpectedStreamCancellationError(LettaError):"""Error raised when a streaming request is terminated unexpectedly."""diff --git a/letta/server/rest_api/app.py b/letta/server/rest_api/app.pyindex 6622cc2c9..7cda044d3 100644--- a/letta/server/rest_api/app.py+++ b/letta/server/rest_api/app.py@@ -31,6 +31,7 @@ from letta.errors import (LettaInvalidMCPSchemaError,LettaMCPConnectionError,LettaMCPTimeoutError,+ LettaServiceUnavailableError,LettaToolCreateError,LettaToolNameConflictError,LettaUnsupportedFileUploadError,@@ -39,6 +40,7 @@ from letta.errors import (LLMError,LLMRateLimitError,LLMTimeoutError,+ PendingApprovalError,)from letta.helpers.pinecone_utils import get_pinecone_indices, should_use_pinecone, upsert_pinecone_indicesfrom letta.jobs.scheduler import start_scheduler_with_leader_election@@ -274,6 +276,7 @@ def create_application() -> "FastAPI":app.add_exception_handler(ForeignKeyConstraintViolationError, _error_handler_409)app.add_exception_handler(UniqueConstraintViolationError, _error_handler_409)app.add_exception_handler(IntegrityError, _error_handler_409)+ app.add_exception_handler(PendingApprovalError, _error_handler_409)# 415 Unsupported Media Type errorsapp.add_exception_handler(LettaUnsupportedFileUploadError, _error_handler_415)@@ -287,6 +290,7 @@ def create_application() -> "FastAPI":# 503 Service Unavailable errorsapp.add_exception_handler(OperationalError, _error_handler_503)+ app.add_exception_handler(LettaServiceUnavailableError, _error_handler_503)@app.exception_handler(IncompatibleAgentType)async def handle_incompatible_agent_type(request: Request, exc: IncompatibleAgentType):diff --git a/letta/server/rest_api/routers/openai/chat_completions/__init__.py b/letta/server/rest_api/routers/openai/chat_completions/__init__.pydeleted file mode 100644index e69de29bb..000000000diff --git a/letta/server/rest_api/routers/v1/__init__.py b/letta/server/rest_api/routers/v1/__init__.pyindex 81b3f401f..6935faf82 100644--- a/letta/server/rest_api/routers/v1/__init__.py+++ b/letta/server/rest_api/routers/v1/__init__.py@@ -1,6 +1,7 @@from letta.server.rest_api.routers.v1.agents import router as agents_routerfrom letta.server.rest_api.routers.v1.archives import router as archives_routerfrom letta.server.rest_api.routers.v1.blocks import router as blocks_router+from letta.server.rest_api.routers.v1.chat_completions import router as chat_completions_router, router as openai_chat_completions_routerfrom letta.server.rest_api.routers.v1.embeddings import router as embeddings_routerfrom letta.server.rest_api.routers.v1.folders import router as folders_routerfrom letta.server.rest_api.routers.v1.groups import router as groups_router@@ -26,6 +27,7 @@ ROUTERS = [sources_router,folders_router,agents_router,+ chat_completions_router,groups_router,identities_router,internal_templates_router,@@ -42,4 +44,5 @@ ROUTERS = [messages_router,voice_router,embeddings_router,+ openai_chat_completions_router,]diff --git a/letta/server/rest_api/routers/v1/chat_completions.py b/letta/server/rest_api/routers/v1/chat_completions.pynew file mode 100644index 000000000..1940de24c--- /dev/null+++ b/letta/server/rest_api/routers/v1/chat_completions.py@@ -0,0 +1,146 @@+from typing import Optional, Union++from fastapi import APIRouter, Body, Depends+from fastapi.responses import StreamingResponse+from openai.types.chat import ChatCompletion+from openai.types.chat.chat_completion_message_param import ChatCompletionMessageParam+from pydantic import BaseModel, Field++from letta.errors import LettaInvalidArgumentError+from letta.log import get_logger+from letta.schemas.enums import MessageRole+from letta.schemas.letta_request import LettaStreamingRequest+from letta.schemas.message import MessageCreate+from letta.server.rest_api.dependencies import HeaderParams, get_headers, get_letta_server+from letta.server.server import SyncServer+from letta.services.streaming_service import StreamingService++logger = get_logger(__name__)++router = APIRouter(tags=["chat"])+++class ChatCompletionRequest(BaseModel):+ """OpenAI-compatible chat completion request - exactly matching OpenAI's schema."""++ model: str = Field(..., description="ID of the model to use")+ messages: list[ChatCompletionMessageParam] = Field(..., description="Messages comprising the conversation so far")++ # optional parameters+ temperature: Optional[float] = Field(None, ge=0, le=2, description="Sampling temperature")+ top_p: Optional[float] = Field(None, ge=0, le=1, description="Nucleus sampling parameter")+ n: Optional[int] = Field(1, ge=1, description="Number of chat completion choices to generate")+ stream: Optional[bool] = Field(False, description="Whether to stream back partial progress")+ stop: Optional[Union[str, list[str]]] = Field(None, description="Sequences where the API will stop generating")+ max_tokens: Optional[int] = Field(None, description="Maximum number of tokens to generate")+ presence_penalty: Optional[float] = Field(None, ge=-2, le=2, description="Presence penalty")+ frequency_penalty: Optional[float] = Field(None, ge=-2, le=2, description="Frequency penalty")+ user: Optional[str] = Field(None, description="A unique identifier representing your end-user")+++async def _handle_chat_completion(+ request: ChatCompletionRequest,+ server: SyncServer,+ headers: HeaderParams,+) -> Union[ChatCompletion, StreamingResponse]:+ """+ Internal handler for chat completion logic.++ Args:+ request: OpenAI-compatible chat completion request+ server: Letta server instance+ headers: Request headers with user info++ Returns:+ Streaming or non-streaming chat completion response+ """+ if request.user:+ actor = await server.user_manager.get_actor_or_default_async(actor_id=request.user)+ else:+ actor = await server.user_manager.get_actor_or_default_async(actor_id=headers.actor_id)++ resolved_agent_id = request.model+ if not resolved_agent_id.startswith("agent-"):+ raise LettaInvalidArgumentError(+ f"For this endpoint, the 'model' field should contain an agent ID (format: 'agent-...'). Received: '{resolved_agent_id}'",+ argument_name="model",+ )+ await server.agent_manager.validate_agent_exists_async(resolved_agent_id, actor)++ # convert OpenAI messages to Letta MessageCreate format+ # NOTE: we only process the last user message+ if len(request.messages) > 1:+ logger.warning(+ f"Chat completions endpoint received {len(request.messages)} messages. "+ "Letta maintains conversation state internally, so only the last user message will be processed. "+ "Previous messages are already stored in the agent's memory."+ )++ last_user_message = None+ for msg in reversed(request.messages):+ role = msg.get("role", "user")+ if role == "user":+ last_user_message = msg+ break++ if not last_user_message:+ raise LettaInvalidArgumentError(+ "No user message found in the request. Please include at least one message with role='user'.",+ argument_name="messages",+ )++ letta_messages = [+ MessageCreate(+ role=MessageRole.user,+ content=last_user_message.get("content", ""),+ )+ ]++ letta_request = LettaStreamingRequest(+ messages=letta_messages,+ stream_tokens=True,+ )++ if request.stream:+ streaming_service = StreamingService(server)+ return await streaming_service.create_agent_stream_openai_chat_completions(+ agent_id=resolved_agent_id,+ actor=actor,+ request=letta_request,+ )+ else:+ raise LettaInvalidArgumentError(+ "Non-streaming chat completions not yet implemented. Please set stream=true.",+ argument_name="stream",+ )+++ "/chat/completions",+ response_model=ChatCompletion,+ responses={+ 200: {+ "description": "Successful response",+ "content": {+ "application/json": {"schema": {"$ref": "#/components/schemas/ChatCompletion"}},+ "text/event-stream": {"description": "Server-Sent Events stream (when stream=true)"},+ },+ }+ },+ operation_id="create_chat_completion",+)+async def create_chat_completion(+ request: ChatCompletionRequest = Body(...),+ server: SyncServer = Depends(get_letta_server),+ headers: HeaderParams = Depends(get_headers),+) -> Union[ChatCompletion, StreamingResponse]:+ """+ Create a chat completion using a Letta agent (OpenAI-compatible).++ This endpoint provides full OpenAI API compatibility. The agent is selected based on:+ - The 'model' parameter in the request (should contain an agent ID in format 'agent-...')++ When streaming is enabled (stream=true), the response will be Server-Sent Events+ with ChatCompletionChunk objects.+ """+ return await _handle_chat_completion(request, server, headers)diff --git a/letta/services/agent_manager.py b/letta/services/agent_manager.pyindex 6f2e32d63..2f390069d 100644--- a/letta/services/agent_manager.py+++ b/letta/services/agent_manager.py@@ -1047,6 +1047,23 @@ class AgentManager:archive_ids = [row[0] for row in result.fetchall()]return archive_ids+ @enforce_types+ @trace_method+ async def validate_agent_exists_async(self, agent_id: str, actor: PydanticUser) -> None:+ """+ Validate that an agent exists and user has access to it.+ Lightweight method that doesn't load the full agent object.++ Args:+ agent_id: ID of the agent to validate+ actor: User performing the action++ Raises:+ LettaAgentNotFoundError: If agent doesn't exist or user doesn't have access+ """+ async with db_registry.async_session() as session:+ await validate_agent_exists_async(session, agent_id, actor)+@enforce_types@trace_methodasync def delete_agent_async(self, agent_id: str, actor: PydanticUser) -> None:diff --git a/letta/services/streaming_service.py b/letta/services/streaming_service.pyindex c6ea8e91b..5c7b714c4 100644--- a/letta/services/streaming_service.py+++ b/letta/services/streaming_service.py@@ -1,36 +1,42 @@-"""-Streaming service for handling agent message streaming with various formats.-Provides a unified interface for streaming agent responses with support for-different output formats (Letta native, OpenAI-compatible, etc.)-"""--import asyncioimport json+import timefrom typing import AsyncIterator, Optional, Union+from uuid import uuid4-from fastapi import HTTPExceptionfrom fastapi.responses import StreamingResponse+from openai.types.chat import ChatCompletionChunk+from openai.types.chat.chat_completion_chunk import Choice, ChoiceDeltafrom letta.agents.agent_loop import AgentLoop+from letta.agents.base_agent_v2 import BaseAgentV2from letta.constants import REDIS_RUN_ID_PREFIXfrom letta.data_sources.redis_client import NoopAsyncRedisClient, get_redis_client-from letta.errors import LLMAuthenticationError, LLMError, LLMRateLimitError, LLMTimeoutError, PendingApprovalError+from letta.errors import (+ LettaInvalidArgumentError,+ LettaServiceUnavailableError,+ LLMAuthenticationError,+ LLMError,+ LLMRateLimitError,+ LLMTimeoutError,+ PendingApprovalError,+)from letta.helpers.datetime_helpers import get_utc_timestamp_nsfrom letta.log import get_loggerfrom letta.otel.context import get_ctx_attributesfrom letta.otel.metric_registry import MetricRegistryfrom letta.schemas.agent import AgentState-from letta.schemas.enums import AgentType, RunStatus+from letta.schemas.enums import AgentType, MessageStreamStatus, RunStatusfrom letta.schemas.job import LettaRequestConfig-from letta.schemas.letta_message import MessageType+from letta.schemas.letta_message import AssistantMessage, MessageType+from letta.schemas.letta_message_content import TextContentfrom letta.schemas.letta_request import LettaStreamingRequestfrom letta.schemas.letta_response import LettaResponsefrom letta.schemas.message import MessageCreatefrom letta.schemas.run import Run as PydanticRun, RunUpdate+from letta.schemas.usage import LettaUsageStatisticsfrom letta.schemas.user import Userfrom letta.server.rest_api.redis_stream_manager import create_background_stream_processor, redis_sse_stream_generatorfrom letta.server.rest_api.streaming_response import StreamingResponseWithStatusCode, add_keepalive_to_stream-from letta.services.lettuce import LettuceClientfrom letta.services.run_manager import RunManagerfrom letta.settings import settingsfrom letta.utils import safe_create_task@@ -116,13 +122,11 @@ class StreamingService:# handle background streaming if requestedif request.background and settings.track_agent_run:if isinstance(redis_client, NoopAsyncRedisClient):- raise HTTPException(- status_code=503,- detail=(- "Background streaming requires Redis to be running. "- "Please ensure Redis is properly configured. "- f"LETTA_REDIS_HOST: {settings.redis_host}, LETTA_REDIS_PORT: {settings.redis_port}"- ),+ raise LettaServiceUnavailableError(+ f"Background streaming requires Redis to be running. "+ f"Please ensure Redis is properly configured. "+ f"LETTA_REDIS_HOST: {settings.redis_host}, LETTA_REDIS_PORT: {settings.redis_port}",+ service_name="redis",)safe_create_task(@@ -176,9 +180,7 @@ class StreamingService:if settings.track_agent_run:run_update_metadata = {"error": str(e)}run_status = RunStatus.failed- raise HTTPException(- status_code=409, detail={"code": "PENDING_APPROVAL", "message": str(e), "pending_request_id": e.pending_request_id}- )+ raiseexcept Exception as e:if settings.track_agent_run:run_update_metadata = {"error": str(e)}@@ -192,9 +194,67 @@ class StreamingService:actor=actor,)+ async def create_agent_stream_openai_chat_completions(+ self,+ agent_id: str,+ actor: User,+ request: LettaStreamingRequest,+ ) -> StreamingResponse:+ """+ Create OpenAI-compatible chat completions streaming response.++ Transforms Letta's internal streaming format to match OpenAI's+ ChatCompletionChunk schema, filtering out internal tool execution+ and only streaming assistant text responses.++ Args:+ agent_id: The agent ID to stream from+ actor: The user making the request+ request: The LettaStreamingRequest containing all request parameters++ Returns:+ StreamingResponse with OpenAI-formatted SSE chunks+ """+ # load agent to get model info for the completion chunks+ agent = await self.server.agent_manager.get_agent_by_id_async(agent_id, actor)++ # create standard Letta stream (returns SSE-formatted stream)+ run, letta_stream_response = await self.create_agent_stream(+ agent_id=agent_id,+ actor=actor,+ request=request,+ run_type="openai_chat_completions",+ )++ # extract the stream iterator from the response+ if isinstance(letta_stream_response, StreamingResponseWithStatusCode):+ letta_stream = letta_stream_response.body_iterator+ else:+ raise LettaInvalidArgumentError(+ "Agent is not compatible with streaming mode",+ argument_name="model",+ )++ # create transformer with agent's model info+ model_name = agent.llm_config.model if agent.llm_config else "unknown"+ completion_id = f"chatcmpl-{run.id if run else str(uuid4())}"++ transformer = OpenAIChatCompletionsStreamTransformer(+ model=model_name,+ completion_id=completion_id,+ )++ # transform Letta SSE stream to OpenAI format (parser handles SSE strings)+ openai_stream = transformer.transform_stream(letta_stream)++ return StreamingResponse(+ openai_stream,+ media_type="text/event-stream",+ )+def _create_error_aware_stream(self,- agent_loop: AgentLoop,+ agent_loop: BaseAgentV2,messages: list[MessageCreate],max_steps: int,stream_tokens: bool,@@ -223,6 +283,7 @@ class StreamingService:request_start_timestamp_ns=request_start_timestamp_ns,include_return_message_types=include_return_message_types,)+async for chunk in stream:yield chunk@@ -336,3 +397,185 @@ class StreamingService:update=update,actor=actor,)+++class OpenAIChatCompletionsStreamTransformer:+ """+ Transforms Letta streaming messages into OpenAI ChatCompletionChunk format.+ Filters out internal tool execution and only streams assistant text responses.+ """++ def __init__(self, model: str, completion_id: str):+ """+ Initialize the transformer.++ Args:+ model: Model name to include in chunks+ completion_id: Unique ID for this completion (format: chatcmpl-{uuid})+ """+ self.model = model+ self.completion_id = completion_id+ self.first_chunk = True+ self.created = int(time.time())++ # TODO: This is lowkey really ugly and poor code design, but this works fine for now+ def _parse_sse_chunk(self, sse_string: str):+ """+ Parse SSE-formatted string back into a message object.++ Args:+ sse_string: SSE formatted string like "data: {...}\n\n"++ Returns:+ Parsed message object or None if can't parse+ """+ try:+ # strip SSE formatting+ if sse_string.startswith("data: "):+ json_str = sse_string[6:].strip()++ # handle [DONE] marker+ if json_str == "[DONE]":+ return MessageStreamStatus.done++ # parse JSON+ data = json.loads(json_str)++ # reconstruct message object based on message_type+ message_type = data.get("message_type")++ if message_type == "assistant_message":+ return AssistantMessage(**data)+ elif message_type == "usage_statistics":+ return LettaUsageStatistics(**data)+ elif message_type == "stop_reason":+ # skip stop_reason, we use [DONE] instead+ return None+ else:+ # other message types we skip+ return None+ return None+ except Exception as e:+ logger.warning(f"Failed to parse SSE chunk: {e}")+ return None++ async def transform_stream(self, letta_stream: AsyncIterator) -> AsyncIterator[str]:+ """+ Transform Letta stream to OpenAI ChatCompletionChunk SSE format.++ Args:+ letta_stream: Async iterator of Letta messages (may be SSE strings or objects)++ Yields:+ SSE-formatted strings: "data: {json}\n\n"+ """+ try:+ async for raw_chunk in letta_stream:+ # parse SSE string if needed+ if isinstance(raw_chunk, str):+ chunk = self._parse_sse_chunk(raw_chunk)+ if chunk is None:+ continue # skip unparseable or filtered chunks+ else:+ chunk = raw_chunk++ # only process assistant messages+ if isinstance(chunk, AssistantMessage):+ async for sse_chunk in self._process_assistant_message(chunk):+ print(f"CHUNK: {sse_chunk}")+ yield sse_chunk++ # handle completion status+ elif chunk == MessageStreamStatus.done:+ # emit final chunk with finish_reason+ final_chunk = ChatCompletionChunk(+ id=self.completion_id,+ object="chat.completion.chunk",+ created=self.created,+ model=self.model,+ choices=[+ Choice(+ index=0,+ delta=ChoiceDelta(),+ finish_reason="stop",+ )+ ],+ )+ yield f"data: {final_chunk.model_dump_json()}\n\n"+ yield "data: [DONE]\n\n"++ except Exception as e:+ logger.error(f"Error in OpenAI stream transformation: {e}", exc_info=True)+ error_chunk = {"error": {"message": str(e), "type": "server_error"}}+ yield f"data: {json.dumps(error_chunk)}\n\n"++ async def _process_assistant_message(self, message: AssistantMessage) -> AsyncIterator[str]:+ """+ Convert AssistantMessage to OpenAI ChatCompletionChunk(s).++ Args:+ message: Letta AssistantMessage with content++ Yields:+ SSE-formatted chunk strings+ """+ # extract text from content (can be string or list of TextContent)+ text_content = self._extract_text_content(message.content)+ if not text_content:+ return++ # emit role on first chunk only+ if self.first_chunk:+ self.first_chunk = False+ # first chunk includes role+ chunk = ChatCompletionChunk(+ id=self.completion_id,+ object="chat.completion.chunk",+ created=self.created,+ model=self.model,+ choices=[+ Choice(+ index=0,+ delta=ChoiceDelta(role="assistant", content=text_content),+ finish_reason=None,+ )+ ],+ )+ else:+ # subsequent chunks just have content+ chunk = ChatCompletionChunk(+ id=self.completion_id,+ object="chat.completion.chunk",+ created=self.created,+ model=self.model,+ choices=[+ Choice(+ index=0,+ delta=ChoiceDelta(content=text_content),+ finish_reason=None,+ )+ ],+ )++ yield f"data: {chunk.model_dump_json()}\n\n"++ def _extract_text_content(self, content: Union[str, list[TextContent]]) -> str:+ """+ Extract text string from content field.++ Args:+ content: Either a string or list of TextContent objects++ Returns:+ Extracted text string+ """+ if isinstance(content, str):+ return content+ elif isinstance(content, list):+ # concatenate all TextContent items+ text_parts = []+ for item in content:+ if isinstance(item, TextContent):+ text_parts.append(item.text)+ return "".join(text_parts)+ return ""diff --git a/tests/integration_test_chat_completions.py b/tests/integration_test_chat_completions.pyindex 97de8babc..896693503 100644--- a/tests/integration_test_chat_completions.py+++ b/tests/integration_test_chat_completions.py@@ -1,6 +1,7 @@import osimport threadingimport uuid+from typing import Listimport pytestfrom dotenv import load_dotenv@@ -9,8 +10,9 @@ from openai import AsyncOpenAIfrom openai.types.chat.chat_completion_chunk import ChatCompletionChunkfrom letta.schemas.embedding_config import EmbeddingConfig-from letta.schemas.enums import MessageStreamStatus+from letta.schemas.enums import AgentType, MessageStreamStatusfrom letta.schemas.llm_config import LLMConfig+from letta.schemas.message import MessageCreatefrom letta.schemas.openai.chat_completion_request import ChatCompletionRequest, UserMessage as OpenAIUserMessagefrom letta.schemas.usage import LettaUsageStatisticsfrom tests.utils import wait_for_server@@ -71,7 +73,7 @@ def weather_tool(client):"""Fetches the current weather for a given location.- Parameters:+ Args:location (str): The location to get the weather for.Returns:@@ -100,6 +102,7 @@ def weather_tool(client):def agent(client, roll_dice_tool, weather_tool):"""Creates an agent and ensures cleanup after tests."""agent_state = client.agents.create(+ agent_type=AgentType.letta_v1_agent,name=f"test_compl_{str(uuid.uuid4())[5:]}",tool_ids=[roll_dice_tool.id, weather_tool.id],include_base_tools=True,@@ -111,7 +114,6 @@ def agent(client, roll_dice_tool, weather_tool):embedding_config=EmbeddingConfig.default_config(provider="openai"),)yield agent_state- client.agents.delete(agent_state.id)# --- Helper Functions --- #@@ -149,42 +151,46 @@ def _assert_valid_chunk(chunk, idx, chunks):@pytest.mark.asyncio[email protected]("message", ["Tell me something interesting about bananas.", "What's the weather in SF?"])[email protected]("endpoint", ["openai/v1"])-async def test_chat_completions_streaming_openai_client(disable_e2b_api_key, client, agent, message, endpoint):- """Tests chat completion streaming using the Async OpenAI client."""- request = _get_chat_request(message)-- async_client = AsyncOpenAI(base_url=f"http://localhost:8283/{endpoint}/{agent.id}", max_retries=0)- stream = await async_client.chat.completions.create(**request.model_dump(exclude_none=True))[email protected]("message", ["Tell me a short joke"])+async def test_chat_completions_streaming_openai_client(disable_e2b_api_key, client, agent, roll_dice_tool, message):+ """Tests Letta's OpenAI-compatible chat completions streaming endpoint."""+ async_client = AsyncOpenAI(base_url="http://localhost:8283/v1", max_retries=0)++ stream = await async_client.chat.completions.create(+ model=agent.id, # agent ID goes in model field+ messages=[{"role": "user", "content": message}],+ stream=True,+ )received_chunks = 0stop_chunk_count = 0last_chunk = None+ content_parts = []try:- async with stream:- async for chunk in stream:- assert isinstance(chunk, ChatCompletionChunk), f"Unexpected chunk type: {type(chunk)}"- assert chunk.choices, "Each ChatCompletionChunk should have at least one choice."-- # Track last chunk for final verification- last_chunk = chunk-- # If this chunk has a finish reason of "stop", track it- if chunk.choices[0].finish_reason == "stop":- stop_chunk_count += 1- # Fail early if more than one stop chunk is sent- assert stop_chunk_count == 1, f"Multiple stop chunks detected: {chunk.model_dump_json(indent=4)}"- continue-- # Validate regular content chunks- assert chunk.choices[0].delta.content, f"Chunk at index {received_chunks} has no content: {chunk.model_dump_json(indent=4)}"+ async for chunk in stream:+ assert isinstance(chunk, ChatCompletionChunk), f"Unexpected chunk type: {type(chunk)}"+ assert chunk.choices, "Each ChatCompletionChunk should have at least one choice."++ last_chunk = chunk++ if chunk.choices[0].finish_reason == "stop":+ stop_chunk_count += 1+ assert stop_chunk_count == 1, f"Multiple stop chunks detected: {chunk.model_dump_json(indent=4)}"+ continue++ if chunk.choices[0].delta.content:+ content_parts.append(chunk.choices[0].delta.content)received_chunks += 1except Exception as e:pytest.fail(f"Streaming failed with exception: {e}")- assert received_chunks > 0, "No valid streaming chunks were received."+ print("\n=== Stream Summary ===")+ print(f"Received chunks: {received_chunks}")+ print(f"Full response: {''.join(content_parts)}")+ print(f"Stop chunk count: {stop_chunk_count}")- # Ensure the last chunk is the expected stop chunk+ assert received_chunks > 0, "No valid streaming chunks were received."+ assert stop_chunk_count == 1, "Expected exactly one stop chunk."assert last_chunk is not None, "No last chunk received."+ assert last_chunk.choices[0].finish_reason == "stop", "Last chunk should have finish_reason='stop'"