Platform for stateful agents: AI with advanced memory that can learn and self-improve over time.
fix: sanitize null bytes to prevent PostgreSQL CharacterNotInRepertoireError (#8015)
github-actions[bot]dbdd1a40e45b
diff --git a/letta/helpers/converters.py b/letta/helpers/converters.pyindex f9befaa99..424dad81b 100644--- a/letta/helpers/converters.py+++ b/letta/helpers/converters.py@@ -6,6 +6,10 @@ from openai.types.chat.chat_completion_message_tool_call import ChatCompletionMefrom sqlalchemy import Dialectfrom letta.functions.mcp_client.types import StdioServerConfig+from letta.helpers.json_helpers import sanitize_null_bytes+from letta.log import get_logger++logger = get_logger(__name__)from letta.schemas.embedding_config import EmbeddingConfigfrom letta.schemas.enums import ProviderType, ToolRuleTypefrom letta.schemas.letta_message import ApprovalReturn, MessageReturnType@@ -184,16 +188,22 @@ def deserialize_tool_rule(def serialize_tool_calls(tool_calls: Optional[List[Union[OpenAIToolCall, dict]]]) -> List[Dict]:- """Convert a list of OpenAI ToolCall objects into JSON-serializable format."""+ """Convert a list of OpenAI ToolCall objects into JSON-serializable format.++ Note: Tool call arguments may contain null bytes from various sources.+ These are sanitized to prevent PostgreSQL errors.+ """if not tool_calls:return []serialized_calls = []for call in tool_calls:if isinstance(call, OpenAIToolCall):- serialized_calls.append(call.model_dump(mode="json"))+ # Sanitize null bytes from tool call data to prevent PostgreSQL errors+ serialized_calls.append(sanitize_null_bytes(call.model_dump(mode="json")))elif isinstance(call, dict):- serialized_calls.append(call) # Already a dictionary, leave it as-is+ # Sanitize null bytes from dictionary data+ serialized_calls.append(sanitize_null_bytes(call))else:raise TypeError(f"Unexpected tool call type: {type(call)}")@@ -221,16 +231,22 @@ def deserialize_tool_calls(data: Optional[List[Dict]]) -> List[OpenAIToolCall]:def serialize_tool_returns(tool_returns: Optional[List[Union[ToolReturn, dict]]]) -> List[Dict]:- """Convert a list of ToolReturn objects into JSON-serializable format."""+ """Convert a list of ToolReturn objects into JSON-serializable format.++ Note: Tool returns may contain null bytes from sandbox execution or binary data.+ These are sanitized to prevent PostgreSQL errors.+ """if not tool_returns:return []serialized_tool_returns = []for tool_return in tool_returns:if isinstance(tool_return, ToolReturn):- serialized_tool_returns.append(tool_return.model_dump(mode="json"))+ # Sanitize null bytes from tool return data to prevent PostgreSQL errors+ serialized_tool_returns.append(sanitize_null_bytes(tool_return.model_dump(mode="json")))elif isinstance(tool_return, dict):- serialized_tool_returns.append(tool_return) # Already a dictionary, leave it as-is+ # Sanitize null bytes from dictionary data+ serialized_tool_returns.append(sanitize_null_bytes(tool_return))else:raise TypeError(f"Unexpected tool return type: {type(tool_return)}")@@ -256,18 +272,24 @@ def deserialize_tool_returns(data: Optional[List[Dict]]) -> List[ToolReturn]:def serialize_approvals(approvals: Optional[List[Union[ApprovalReturn, ToolReturn, dict]]]) -> List[Dict]:- """Convert a list of ToolReturn objects into JSON-serializable format."""+ """Convert a list of ToolReturn objects into JSON-serializable format.++ Note: Approval data may contain null bytes from various sources.+ These are sanitized to prevent PostgreSQL errors.+ """if not approvals:return []serialized_approvals = []for approval in approvals:if isinstance(approval, ApprovalReturn):- serialized_approvals.append(approval.model_dump(mode="json"))+ # Sanitize null bytes from approval data to prevent PostgreSQL errors+ serialized_approvals.append(sanitize_null_bytes(approval.model_dump(mode="json")))elif isinstance(approval, ToolReturn):- serialized_approvals.append(approval.model_dump(mode="json"))+ serialized_approvals.append(sanitize_null_bytes(approval.model_dump(mode="json")))elif isinstance(approval, dict):- serialized_approvals.append(approval) # Already a dictionary, leave it as-is+ # Sanitize null bytes from dictionary data+ serialized_approvals.append(sanitize_null_bytes(approval))else:raise TypeError(f"Unexpected approval type: {type(approval)}")@@ -318,7 +340,11 @@ def deserialize_approvals(data: Optional[List[Dict]]) -> List[Union[ApprovalRetudef serialize_message_content(message_content: Optional[List[Union[MessageContent, dict]]]) -> List[Dict]:- """Convert a list of MessageContent objects into JSON-serializable format."""+ """Convert a list of MessageContent objects into JSON-serializable format.++ Note: Message content may contain null bytes from various sources.+ These are sanitized to prevent PostgreSQL errors.+ """if not message_content:return []@@ -327,9 +353,11 @@ def serialize_message_content(message_content: Optional[List[Union[MessageContenif isinstance(content, MessageContent):if content.type == MessageContentType.image:assert content.source.type == ImageSourceType.letta, f"Invalid image source type: {content.source.type}"- serialized_message_content.append(content.model_dump(mode="json"))+ # Sanitize null bytes from message content to prevent PostgreSQL errors+ serialized_message_content.append(sanitize_null_bytes(content.model_dump(mode="json")))elif isinstance(content, dict):- serialized_message_content.append(content) # Already a dictionary, leave it as-is+ # Sanitize null bytes from dictionary data+ serialized_message_content.append(sanitize_null_bytes(content))else:raise TypeError(f"Unexpected message content type: {type(content)}")return serialized_message_contentdiff --git a/letta/helpers/json_helpers.py b/letta/helpers/json_helpers.pyindex ff6943b8f..aba17d05d 100644--- a/letta/helpers/json_helpers.py+++ b/letta/helpers/json_helpers.py@@ -1,6 +1,42 @@import base64import jsonfrom datetime import datetime+from typing import Any+++def sanitize_null_bytes(value: Any) -> Any:+ """Recursively remove null bytes (0x00) from strings.++ PostgreSQL TEXT columns don't accept null bytes in UTF-8 encoding, which causes+ asyncpg.exceptions.CharacterNotInRepertoireError when data with null bytes is inserted.++ This function sanitizes:+ - Strings: removes all null bytes+ - Dicts: recursively sanitizes all string values+ - Lists: recursively sanitizes all elements+ - Other types: returned as-is++ Args:+ value: The value to sanitize++ Returns:+ The sanitized value with null bytes removed from all strings+ """+ if isinstance(value, str):+ # Remove null bytes from strings+ return value.replace("\x00", "")+ elif isinstance(value, dict):+ # Recursively sanitize dictionary values+ return {k: sanitize_null_bytes(v) for k, v in value.items()}+ elif isinstance(value, list):+ # Recursively sanitize list elements+ return [sanitize_null_bytes(item) for item in value]+ elif isinstance(value, tuple):+ # Recursively sanitize tuple elements (return as tuple)+ return tuple(sanitize_null_bytes(item) for item in value)+ else:+ # Return other types as-is (int, float, bool, None, etc.)+ return valuedef json_loads(data):@@ -8,15 +44,33 @@ def json_loads(data):def json_dumps(data, indent=2) -> str:+ """Serialize data to JSON string, sanitizing null bytes to prevent PostgreSQL errors.++ PostgreSQL TEXT columns reject null bytes (0x00) in UTF-8 encoding. This function+ sanitizes all strings in the data structure before JSON serialization to prevent+ asyncpg.exceptions.CharacterNotInRepertoireError.++ Args:+ data: The data to serialize+ indent: JSON indentation level (default: 2)++ Returns:+ JSON string with null bytes removed from all string values+ """+ # Sanitize null bytes before serialization to prevent PostgreSQL errors+ sanitized_data = sanitize_null_bytes(data)+def safe_serializer(obj):if isinstance(obj, datetime):return obj.isoformat()if isinstance(obj, bytes):try:- return obj.decode("utf-8")+ decoded = obj.decode("utf-8")+ # Also sanitize decoded bytes+ return decoded.replace("\x00", "")except Exception:# TODO: this is to handle Gemini thought signatures, b64 decode this back to bytes when sending back to Geminireturn base64.b64encode(obj).decode("utf-8")raise TypeError(f"Type {type(obj)} not serializable")- return json.dumps(data, indent=indent, default=safe_serializer, ensure_ascii=False)+ return json.dumps(sanitized_data, indent=indent, default=safe_serializer, ensure_ascii=False)diff --git a/tests/test_utils.py b/tests/test_utils.pyindex 3d16039e2..37114aa7b 100644--- a/tests/test_utils.py+++ b/tests/test_utils.py@@ -678,3 +678,102 @@ def test_sdk_version_check():assert is_1_0_sdk_version(HeaderParams(sdk_version="v1.0.0-alpha.7"))assert is_1_0_sdk_version(HeaderParams(sdk_version="v1.0.0a7"))assert is_1_0_sdk_version(HeaderParams(sdk_version="v2.0.0"))+++# ---------------------- sanitize_null_bytes TESTS ---------------------- #+++def test_sanitize_null_bytes_string():+ """Test that null bytes are removed from strings"""+ from letta.helpers.json_helpers import sanitize_null_bytes++ # Test basic null byte removal+ assert sanitize_null_bytes("hello\x00world") == "helloworld"++ # Test multiple null bytes+ assert sanitize_null_bytes("a\x00b\x00c") == "abc"++ # Test null byte at beginning+ assert sanitize_null_bytes("\x00hello") == "hello"++ # Test null byte at end+ assert sanitize_null_bytes("hello\x00") == "hello"++ # Test string without null bytes+ assert sanitize_null_bytes("hello world") == "hello world"++ # Test empty string+ assert sanitize_null_bytes("") == ""+++def test_sanitize_null_bytes_dict():+ """Test that null bytes are removed from dictionary values"""+ from letta.helpers.json_helpers import sanitize_null_bytes++ # Test nested dict with null bytes+ result = sanitize_null_bytes({+ "key1": "value\x00with\x00nulls",+ "key2": {"nested": "also\x00null"},+ "key3": 123, # non-string should be unchanged+ })+ assert result == {+ "key1": "valuewithnulls",+ "key2": {"nested": "alsonull"},+ "key3": 123,+ }+++def test_sanitize_null_bytes_list():+ """Test that null bytes are removed from list elements"""+ from letta.helpers.json_helpers import sanitize_null_bytes++ result = sanitize_null_bytes(["hello\x00world", "no nulls", {"nested\x00key": "value\x00"}])+ assert result == ["helloworld", "no nulls", {"nestedkey": "value"}]+++def test_sanitize_null_bytes_tuple():+ """Test that null bytes are removed from tuple elements"""+ from letta.helpers.json_helpers import sanitize_null_bytes++ result = sanitize_null_bytes(("hello\x00world", "no nulls"))+ assert result == ("helloworld", "no nulls")+++def test_sanitize_null_bytes_preserves_other_types():+ """Test that non-string types are preserved unchanged"""+ from letta.helpers.json_helpers import sanitize_null_bytes++ assert sanitize_null_bytes(123) == 123+ assert sanitize_null_bytes(3.14) == 3.14+ assert sanitize_null_bytes(True) is True+ assert sanitize_null_bytes(False) is False+ assert sanitize_null_bytes(None) is None+++def test_json_dumps_sanitizes_null_bytes():+ """Test that json_dumps sanitizes null bytes before serialization"""+ from letta.helpers.json_helpers import json_dumps++ # Test that null bytes are removed from the output+ result = json_dumps({"message": "hello\x00world"})+ assert "\x00" not in result+ assert "helloworld" in result+++def test_json_dumps_with_complex_nested_null_bytes():+ """Test that json_dumps handles complex nested structures with null bytes"""+ from letta.helpers.json_helpers import json_dumps++ data = {+ "tool_return": {+ "status": "success",+ "func_response": "Binary\x00data\x00here",+ },+ "content": [+ {"type": "text", "text": "Message\x00with\x00nulls"},+ ],+ }+ result = json_dumps(data)+ assert "\x00" not in result+ assert "Binarydatahere" in result+ assert "Messagewithnulls" in result