Platform for stateful agents: AI with advanced memory that can learn and self-improve over time.
fix: remove db locks (#4879)
Sarah Wooders8fee5b4b9303
diff --git a/letta/server/db.py b/letta/server/db.pyindex 8a98b1398..4c1e32232 100644--- a/letta/server/db.py+++ b/letta/server/db.py@@ -1,433 +1,111 @@-import asyncio-import os-import threading-import timeimport uuid-from contextlib import asynccontextmanager, contextmanager-from typing import Any, AsyncGenerator, Generator--from opentelemetry import trace-from rich.console import Console-from rich.panel import Panel-from rich.text import Text-from sqlalchemy import Engine, NullPool, QueuePool, create_engine, event-from sqlalchemy.ext.asyncio import AsyncEngine, AsyncSession, async_sessionmaker, create_async_engine-from sqlalchemy.orm import sessionmaker--from letta.config import LettaConfig-from letta.log import get_logger-from letta.otel.tracing import trace_method-from letta.settings import DatabaseChoice, settings--logger = get_logger(__name__)---def print_sqlite_schema_error():- """Print a formatted error message for SQLite schema issues"""- console = Console()- error_text = Text()- error_text.append("Existing SQLite DB schema is invalid, and schema migrations are not supported for SQLite. ", style="bold red")- error_text.append("To have migrations supported between Letta versions, please run Letta with Docker (", style="white")- error_text.append("https://docs.letta.com/server/docker", style="blue underline")- error_text.append(") or use Postgres by setting ", style="white")- error_text.append("LETTA_PG_URI", style="yellow")- error_text.append(".\n\n", style="white")- error_text.append("If you wish to keep using SQLite, you can reset your database by removing the DB file with ", style="white")- error_text.append("rm ~/.letta/sqlite.db", style="yellow")- error_text.append(" or downgrade to your previous version of Letta.", style="white")-- console.print(Panel(error_text, border_style="red"))--[email protected]_for(Engine, "connect")-def enable_sqlite_foreign_keys(dbapi_connection, connection_record):- """Enable foreign key constraints for SQLite connections."""- if "sqlite" in str(dbapi_connection):- cursor = dbapi_connection.cursor()- cursor.execute("PRAGMA foreign_keys=ON")- cursor.close()---def on_connect(dbapi_connection, connection_record):- cursor = dbapi_connection.cursor()- cursor.execute("SELECT pg_backend_pid()")- pid = cursor.fetchone()[0]- connection_record.info["pid"] = pid- connection_record.info["connect_spawn_time_ms"] = time.perf_counter() * 1000- cursor.close()---def on_close(dbapi_connection, connection_record):- connection_record.info.get("pid")- (time.perf_counter() * 1000) - connection_record.info.get("connect_spawn_time_ms")- # print(f"Connection closed: {pid}, duration: {duration:.6f}s")---def on_checkout(dbapi_connection, connection_record, connection_proxy):- connection_record.info.get("pid")- connection_record.info["connect_checkout_time_ms"] = time.perf_counter() * 1000---def on_checkin(dbapi_connection, connection_record):- pid = connection_record.info.get("pid")- duration = (time.perf_counter() * 1000) - connection_record.info.get("connect_checkout_time_ms")-- tracer = trace.get_tracer("letta.db.connection")- with tracer.start_as_current_span("connect_release") as span:- span.set_attribute("db.connection.pid", pid)- span.set_attribute("db.connection.duration_ms", duration)- span.set_attribute("db.connection.operation", "checkin")---@contextmanager-def db_error_handler():- """Context manager for handling database errors"""- try:- yield- except Exception as e:- # Handle other SQLAlchemy errors- error_str = str(e)-- # Don't exit for expected constraint violations that should be handled by the application- if "UNIQUE constraint failed" in error_str or "FOREIGN KEY constraint failed" in error_str:- # These are application-level errors that should be handled by the ORM- raise-- # For other database errors, print error and exit- print(e)- print_sqlite_schema_error()- # raise ValueError(f"SQLite DB error: {str(e)}")- exit(1)---class DatabaseRegistry:- """Registry for database connections and sessions.-- This class manages both synchronous and asynchronous database connections- and provides context managers for session handling.- """-- def __init__(self):- # self._engines: dict[str, Engine] = {}- self._async_engines: dict[str, AsyncEngine] = {}- # self._session_factories: dict[str, sessionmaker] = {}- self._async_session_factories: dict[str, async_sessionmaker] = {}- self._initialized: dict[str, bool] = {"sync": False, "async": False}- self._lock = threading.Lock()- self.config = LettaConfig.load()- self.logger = get_logger(__name__)-- if settings.db_max_concurrent_sessions:- self._db_semaphore = asyncio.Semaphore(settings.db_max_concurrent_sessions)- self.logger.info(f"Initialized database throttling with max {settings.db_max_concurrent_sessions} concurrent sessions")- else:- self.logger.info("Database throttling is disabled")- self._db_semaphore = None-- # def initialize_sync(self, force: bool = False) -> None:- # """Initialize the synchronous database engine if not already initialized."""- # with self._lock:- # if self._initialized.get("sync") and not force:- # return-- # # Postgres engine- # if settings.database_engine is DatabaseChoice.POSTGRES:- # self.logger.info("Creating postgres engine")- # self.config.recall_storage_type = "postgres"- # self.config.recall_storage_uri = settings.letta_pg_uri_no_default- # self.config.archival_storage_type = "postgres"- # self.config.archival_storage_uri = settings.letta_pg_uri_no_default-- # engine = create_engine(settings.letta_pg_uri, **self._build_sqlalchemy_engine_args(is_async=False))-- # self._engines["default"] = engine- # # SQLite engine- # else:- # from letta.orm import Base-- # # TODO: don't rely on config storage- # engine_path = "sqlite:///" + os.path.join(self.config.recall_storage_path, "sqlite.db")- # self.logger.info("Creating sqlite engine " + engine_path)-- # engine = create_engine(engine_path)-- # # Wrap the engine with error handling- # self._wrap_sqlite_engine(engine)-- # Base.metadata.create_all(bind=engine)- # self._engines["default"] = engine-- # # Set up connection monitoring- # if settings.sqlalchemy_tracing and settings.database_engine is DatabaseChoice.POSTGRES:- # event.listen(engine, "connect", on_connect)- # event.listen(engine, "close", on_close)- # event.listen(engine, "checkout", on_checkout)- # event.listen(engine, "checkin", on_checkin)-- # self._setup_pool_monitoring(engine, "default")-- # # Create session factory- # self._session_factories["default"] = sessionmaker(autocommit=False, autoflush=False, bind=self._engines["default"])- # self._initialized["sync"] = True-- def initialize_async(self, force: bool = False) -> None:- """Initialize the asynchronous database engine if not already initialized."""- with self._lock:- if self._initialized.get("async") and not force:- return-- if settings.database_engine is DatabaseChoice.POSTGRES:- self.logger.info("Creating async postgres engine")-- # Create async engine - convert URI to async format- pg_uri = settings.letta_pg_uri- if pg_uri.startswith("postgresql://"):- async_pg_uri = pg_uri.replace("postgresql://", "postgresql+asyncpg://")- else:- async_pg_uri = f"postgresql+asyncpg://{pg_uri.split('://', 1)[1]}" if "://" in pg_uri else pg_uri- async_pg_uri = async_pg_uri.replace("sslmode=", "ssl=")- async_engine = create_async_engine(async_pg_uri, **self._build_sqlalchemy_engine_args(is_async=True))- else:- # create sqlite async engine- self._initialized["async"] = False- # TODO: remove self.config- engine_path = "sqlite+aiosqlite:///" + os.path.join(self.config.recall_storage_path, "sqlite.db")- self.logger.info("Creating sqlite engine " + engine_path)- async_engine = create_async_engine(engine_path, **self._build_sqlalchemy_engine_args(is_async=True))-- # Enable foreign keys for SQLite async connections- @event.listens_for(async_engine.sync_engine, "connect")- def enable_sqlite_foreign_keys_async(dbapi_connection, connection_record):- cursor = dbapi_connection.cursor()- cursor.execute("PRAGMA foreign_keys=ON")- cursor.close()-- # Create async session factory- self._async_engines["default"] = async_engine-- # Set up connection monitoring for async engine- if settings.sqlalchemy_tracing and settings.database_engine is DatabaseChoice.POSTGRES:- event.listen(async_engine.sync_engine, "connect", on_connect)- event.listen(async_engine.sync_engine, "close", on_close)- event.listen(async_engine.sync_engine, "checkout", on_checkout)- event.listen(async_engine.sync_engine, "checkin", on_checkin)-- self._setup_pool_monitoring(async_engine, "default_async")-- self._async_session_factories["default"] = async_sessionmaker(- expire_on_commit=False,- close_resets_only=False,- autocommit=False,- autoflush=False,- bind=self._async_engines["default"],- class_=AsyncSession,- )- self._initialized["async"] = True-- def _build_sqlalchemy_engine_args(self, *, is_async: bool) -> dict:- """Prepare keyword arguments for create_engine / create_async_engine."""- # For async SQLite, always use NullPool to avoid cleanup issues during cancellation- if is_async and settings.database_engine is DatabaseChoice.SQLITE:- use_null_pool = True- logger.info("Forcing NullPool for async SQLite to avoid cancellation cleanup issues")- else:- use_null_pool = settings.disable_sqlalchemy_pooling-- if use_null_pool:- logger.info("Disabling pooling on SqlAlchemy")- pool_cls = NullPool- else:- logger.info("Enabling pooling on SqlAlchemy")- # AsyncAdaptedQueuePool will be the default if none is provided for async but setting this explicitly.- from sqlalchemy import AsyncAdaptedQueuePool-- pool_cls = QueuePool if not is_async else AsyncAdaptedQueuePool-- base_args = {- "echo": settings.pg_echo,- "pool_pre_ping": settings.pool_pre_ping,+from contextlib import asynccontextmanager+from typing import AsyncGenerator++from sqlalchemy import NullPool+from sqlalchemy.ext.asyncio import (+ AsyncEngine,+ AsyncSession,+ async_sessionmaker,+ create_async_engine,+)++from letta.settings import settings++# Convert PostgreSQL URI to async format+pg_uri = settings.letta_pg_uri+if pg_uri.startswith("postgresql://"):+ async_pg_uri = pg_uri.replace("postgresql://", "postgresql+asyncpg://")+else:+ # Handle other URI formats (e.g., postgresql+pg8000://)+ async_pg_uri = f"postgresql+asyncpg://{pg_uri.split('://', 1)[1]}" if "://" in pg_uri else pg_uri++# Replace sslmode with ssl for asyncpg+async_pg_uri = async_pg_uri.replace("sslmode=", "ssl=")++# Build engine configuration based on settings+engine_args = {+ "echo": settings.pg_echo,+ "pool_pre_ping": settings.pool_pre_ping,+}++# Configure pooling+if settings.disable_sqlalchemy_pooling:+ engine_args["poolclass"] = NullPool+else:+ # Use default AsyncAdaptedQueuePool with configured settings+ engine_args.update(+ {+ "pool_size": settings.pg_pool_size,+ "max_overflow": settings.pg_max_overflow,+ "pool_timeout": settings.pg_pool_timeout,+ "pool_recycle": settings.pg_pool_recycle,}+ )++# Add asyncpg-specific settings for connection+if not settings.disable_sqlalchemy_pooling:+ engine_args["connect_args"] = {+ "timeout": settings.pg_pool_timeout,+ "prepared_statement_name_func": lambda: f"__asyncpg_{uuid.uuid4()}__",+ "statement_cache_size": 0,+ "prepared_statement_cache_size": 0,+ }++# Create the engine once at module level+engine: AsyncEngine = create_async_engine(async_pg_uri, **engine_args)++# Create session factory once at module level+async_session_factory = async_sessionmaker(+ engine,+ class_=AsyncSession,+ expire_on_commit=False,+ autocommit=False,+ autoflush=False,+)- if pool_cls:- base_args["poolclass"] = pool_cls-- if not use_null_pool:- base_args.update(- {- "pool_size": settings.pg_pool_size,- "max_overflow": settings.pg_max_overflow,- "pool_timeout": settings.pg_pool_timeout,- "pool_recycle": settings.pg_pool_recycle,- }- )- if not is_async:- base_args.update(- {- "pool_use_lifo": settings.pool_use_lifo,- }- )-- elif is_async and settings.database_engine is DatabaseChoice.POSTGRES:- # Invalid for SQLite, results in [0] TypeError: 'prepared_statement_name_func' is an invalid keyword argument for Connection()- # For asyncpg, statement_cache_size should be in connect_args- base_args.update(- {- "connect_args": {- "timeout": settings.pg_pool_timeout,- "prepared_statement_name_func": lambda: f"__asyncpg_{uuid.uuid4()}__",- "statement_cache_size": 0,- "prepared_statement_cache_size": 0,- },- }- )- return base_args-- def _wrap_sqlite_engine(self, engine: Engine) -> None:- """Wrap SQLite engine with error handling."""- original_connect = engine.connect-- def wrapped_connect(*args, **kwargs):- with db_error_handler():- connection = original_connect(*args, **kwargs)- original_execute = connection.execute-- def wrapped_execute(*args, **kwargs):- with db_error_handler():- return original_execute(*args, **kwargs)-- connection.execute = wrapped_execute- return connection-- engine.connect = wrapped_connect-- def _setup_pool_monitoring(self, engine: Engine | AsyncEngine, engine_name: str) -> None:- """Set up database pool monitoring for the given engine."""- if not settings.enable_db_pool_monitoring:- return-- try:- from letta.otel.db_pool_monitoring import setup_pool_monitoring-- setup_pool_monitoring(engine, engine_name)- self.logger.info(f"Database pool monitoring enabled for {engine_name}")- except ImportError:- self.logger.warning("Database pool monitoring not available - missing dependencies")- except Exception as e:- self.logger.warning(f"Failed to setup pool monitoring for {engine_name}: {e}")- # def get_engine(self, name: str = "default") -> Engine:- # """Get a database engine by name."""- # self.initialize_sync()- # return self._engines.get(name)-- def get_async_engine(self, name: str = "default") -> Engine:- """Get a database engine by name."""- self.initialize_async()- return self._async_engines.get(name)-- # def get_session_factory(self, name: str = "default") -> sessionmaker:- # """Get a session factory by name."""- # self.initialize_sync()- # return self._session_factories.get(name)-- def get_async_session_factory(self, name: str = "default") -> async_sessionmaker:- """Get an async session factory by name."""- self.initialize_async()- return self._async_session_factories.get(name)-- # @trace_method- # @contextmanager- # def session(self, name: str = "default") -> Generator[Any, None, None]:- # """Context manager for database sessions."""- # caller_info = "unknown caller"- # try:- # import inspect-- # frame = inspect.currentframe()- # stack = inspect.getouterframes(frame)-- # for i, frame_info in enumerate(stack):- # module = inspect.getmodule(frame_info.frame)- # module_name = module.__name__ if module else "unknown"-- # if module_name != "contextlib" and "db.py" not in frame_info.filename:- # caller_module = module_name- # caller_function = frame_info.function- # caller_lineno = frame_info.lineno- # caller_file = frame_info.filename.split("/")[-1]-- # caller_info = f"{caller_module}.{caller_function}:{caller_lineno} ({caller_file})"- # break- # except:- # pass- # finally:- # del frame-- # self.session_caller_trace(caller_info)-- # session_factory = self.get_session_factory(name)- # if not session_factory:- # raise ValueError(f"No session factory found for '{name}'")-- # session = session_factory()- # try:- # yield session- # finally:- # session.close()+class DatabaseRegistry:+ """Dummy registry to maintain the existing interface."""- @trace_method@asynccontextmanager- async def async_session(self, name: str = "default") -> AsyncGenerator[AsyncSession, None]:- """Async context manager for database sessions with throttling."""- if self._db_semaphore:- async with self._db_semaphore:- session_factory = self.get_async_session_factory(name)- if not session_factory:- raise ValueError(f"No async session factory found for '{name}' or async database is not configured")-- session = session_factory()- try:- yield session- finally:- await session.close()- else:- session_factory = self.get_async_session_factory(name)- if not session_factory:- raise ValueError(f"No async session factory found for '{name}' or async database is not configured")-- session = session_factory()+ async def async_session(self) -> AsyncGenerator[AsyncSession, None]:+ """Get an async database session."""+ async with async_session_factory() as session:try:yield session+ await session.commit()+ except Exception:+ await session.rollback()+ raisefinally:await session.close()- @trace_method- def session_caller_trace(self, caller_info: str):- """Trace sync db caller information for debugging purposes."""- pass # wrapper used for otel tracing only--# Create a singleton instance+# Create singleton instance to match existing interfacedb_registry = DatabaseRegistry()+# Backwards compatibility functiondef get_db_registry() -> DatabaseRegistry:"""Get the global database registry instance."""return db_registry-# def get_db():-# """Get a database session."""-# with db_registry.session() as session:-# yield session---async def get_db_async():+# FastAPI dependency helper+async def get_db_async() -> AsyncGenerator[AsyncSession, None]:"""Get an async database session."""async with db_registry.async_session() as session:yield session-# Prefer calling db_registry.session() or db_registry.async_session() directly-# This is for backwards compatibility-# db_context = contextmanager(get_db)+# Optional: cleanup function for graceful shutdown+async def close_db() -> None:+ """Close the database engine."""+ await engine.dispose()+++# Usage remains the same:+# async with db_registry.async_session() as session:+# result = await session.execute(select(User))+# users = result.scalars().all()diff --git a/letta/server/rest_api/app.py b/letta/server/rest_api/app.pyindex f354b6142..0d06489f5 100644--- a/letta/server/rest_api/app.py+++ b/letta/server/rest_api/app.py@@ -121,10 +121,10 @@ async def lifespan(app_: FastAPI):except Exception as exc:logger.info("Profiler not enabled: %", exc)- logger.info(f"[Worker {worker_id}] Starting lifespan initialization")- logger.info(f"[Worker {worker_id}] Initializing database connections")- db_registry.initialize_async()- logger.info(f"[Worker {worker_id}] Database connections initialized")+ # logger.info(f"[Worker {worker_id}] Starting lifespan initialization")+ # logger.info(f"[Worker {worker_id}] Initializing database connections")+ # db_registry.initialize_async()+ # logger.info(f"[Worker {worker_id}] Database connections initialized")if should_use_pinecone():if settings.upsert_pinecone_indices: