Skip to content

armgit / burr

Public Imported from GitHub
0

Build applications that make decisions (chatbots, agents, simulations, etc...). Monitor, trace, persist, and execute on your own infrastructure.

Add async PostgreSQL persister support

jernejfrankeec5c7d4e827
diff --git a/burr/integrations/persisters/b_asyncpg.py b/burr/integrations/persisters/b_asyncpg.py
new file mode 100644
index 00000000..d6f788f4
--- /dev/null
+++ b/burr/integrations/persisters/b_asyncpg.py
@@ -0,0 +1,264 @@
+from burr.integrations import base
+
+try:
+ import asyncpg
+except ImportError as e:
+ base.require_plugin(e, "asyncpg")
+
+import json
+import logging
+from typing import Literal, Optional
+
+from burr.core import persistence, state
+
+logger = logging.getLogger(__name__)
+
+
+class AsyncPostgreSQLPersister(persistence.AsyncBaseStatePersister):
+ """Class for async PostgreSQL persistence of state.
+
+ .. warning::
+ The synchronous persister closes the connection on deletion of the class using the ``__del__`` method.
+ In an async context that is not reliable (the event loop may already be closed by the time ``__del__``
+ gets invoked). Therefore, you are responsible for closing the connection yourself (i.e. manual cleanup).
+ We suggest to use the persister either as a context manager through the ``async with`` clause or
+ using the method ``.cleanup()``.
+
+ .. note::
+ The implementation relies on the popular asyncpg library: https://github.com/MagicStack/asyncpg
+
+ MagicStack wrote a blog post: http://magic.io/blog/asyncpg-1m-rows-from-postgres-to-python/
+ explaining why a new implementation, performance review and comparison to aiopg (async interface of psycopg2).
+
+
+ To try it out locally with docker -- here's a command -- change the values as appropriate.
+
+ .. code:: bash
+
+ docker run --name local-psql \ # container name
+ -v local_psql_data:/SOME/FILE_PATH/ \ # mounting a volume for data persistence
+ -p 54320:5432 \ # port mapping
+ -e POSTGRES_PASSWORD=my_password \ # superuser password
+ -d postgres # database name
+
+ Then you should be able to create the class like this:
+
+ .. code:: python
+
+ p = await AsyncPostgreSQLPersister.from_values("postgres", "postgres", "my_password",
+ "localhost", 54320, table_name="burr_state")
+
+
+ """
+
+ PARTITION_KEY_DEFAULT = ""
+
+ @classmethod
+ async def from_config(cls, config: dict) -> "AsyncPostgreSQLPersister":
+ """Creates a new instance of the PostgreSQLPersister from a configuration dictionary."""
+ return await cls.from_values(**config)
+
+ @classmethod
+ async def from_values(
+ cls,
+ db_name: str,
+ user: str,
+ password: str,
+ host: str,
+ port: int,
+ table_name: str = "burr_state",
+ ) -> "AsyncPostgreSQLPersister":
+ """Builds a new instance of the PostgreSQLPersister from the provided values.
+
+ :param db_name: the name of the PostgreSQL database.
+ :param user: the username to connect to the PostgreSQL database.
+ :param password: the password to connect to the PostgreSQL database.
+ :param host: the host of the PostgreSQL database.
+ :param port: the port of the PostgreSQL database.
+ :param table_name: the table name to store things under.
+ """
+ connection = await asyncpg.connect(
+ user=user, password=password, database=db_name, host=host, port=port
+ )
+ return cls(connection, table_name)
+
+ def __init__(self, connection, table_name: str = "burr_state", serde_kwargs: dict = None):
+ """Constructor
+
+ :param connection: the connection to the PostgreSQL database.
+ :param table_name: the table name to store things under.
+ """
+ self.table_name = table_name
+ self.connection = connection
+ self.serde_kwargs = serde_kwargs or {}
+ self._initialized = False
+
+ async def __aenter__(self):
+ return self
+
+ async def __aexit__(self, exc_type, exc_value, traceback):
+ await self.connection.close()
+ return False
+
+ def set_serde_kwargs(self, serde_kwargs: dict):
+ """Sets the serde_kwargs for the persister."""
+ self.serde_kwargs = serde_kwargs
+
+ async def create_table(self, table_name: str):
+ """Helper function to create the table where things are stored."""
+ async with self.connection.transaction():
+ await self.connection.execute(
+ f"""
+ CREATE TABLE IF NOT EXISTS {table_name} (
+ partition_key TEXT DEFAULT '{self.PARTITION_KEY_DEFAULT}',
+ app_id TEXT NOT NULL,
+ sequence_id INTEGER NOT NULL,
+ position TEXT NOT NULL,
+ status TEXT NOT NULL,
+ state JSONB NOT NULL,
+ created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
+ PRIMARY KEY (partition_key, app_id, sequence_id, position)
+ )"""
+ )
+ await self.connection.execute(
+ f"""
+ CREATE INDEX IF NOT EXISTS {table_name}_created_at_index ON {table_name} (created_at);
+ """
+ )
+
+ async def initialize(self):
+ """Creates the table"""
+ await self.create_table(self.table_name)
+ self._initialized = True
+
+ async def is_initialized(self) -> bool:
+ """This checks to see if the table has been created in the database or not.
+ It defaults to using the initialized field, else queries the database to see if the table exists.
+ It then sets the initialized field to True if the table exists.
+ """
+ if self._initialized:
+ return True
+
+ query = "SELECT EXISTS (SELECT FROM information_schema.tables WHERE table_name = $1)"
+ self._initialized = await self.connection.fetchval(query, self.table_name, column=0)
+ return self._initialized
+
+ async def list_app_ids(self, partition_key: str, **kwargs) -> list[str]:
+ """Lists the app_ids for a given partition_key."""
+ query = (
+ f"SELECT DISTINCT app_id, created_at FROM {self.table_name} "
+ "WHERE partition_key = $1 "
+ "ORDER BY created_at DESC"
+ )
+ fetched_data = await self.connection.fetch(query, partition_key)
+ app_ids = [row[0] for row in fetched_data]
+ return app_ids
+
+ async def load(
+ self, partition_key: Optional[str], app_id: str, sequence_id: int = None, **kwargs
+ ) -> Optional[persistence.PersistedStateData]:
+ """Loads state for a given partition id.
+
+ Depending on the parameters, this will return the last thing written, the last thing written for a given app_id,
+ or a specific sequence_id for a given app_id.
+
+ :param partition_key:
+ :param app_id:
+ :param sequence_id:
+ :return:
+ """
+ if partition_key is None:
+ partition_key = self.PARTITION_KEY_DEFAULT
+ logger.debug("Loading %s, %s, %s", partition_key, app_id, sequence_id)
+
+ if app_id is None:
+ # get latest for all app_ids
+ query = (
+ f"SELECT position, state, sequence_id, app_id, created_at, status FROM {self.table_name} "
+ "WHERE partition_key = $1 "
+ f"ORDER BY CREATED_AT DESC LIMIT 1"
+ )
+ row = await self.connection.fetchrow(query, partition_key)
+
+ elif sequence_id is None:
+ query = (
+ f"SELECT position, state, sequence_id, app_id, created_at, status FROM {self.table_name} "
+ "WHERE partition_key = $1 AND app_id = $2 "
+ f"ORDER BY sequence_id DESC LIMIT 1"
+ )
+ row = await self.connection.fetchrow(query, partition_key, app_id)
+ else:
+ query = (
+ f"SELECT position, state, sequence_id, app_id, created_at, status FROM {self.table_name} "
+ "WHERE partition_key = $1 AND app_id = $2 AND sequence_id = $3 "
+ )
+ row = await self.connection.fetchrow(
+ query,
+ partition_key,
+ app_id,
+ sequence_id,
+ )
+ if row is None:
+ return None
+ # converts from asyncpg str to dict
+ json_row = json.loads(row[1])
+ _state = state.State.deserialize(json_row, **self.serde_kwargs)
+ return {
+ "partition_key": partition_key,
+ "app_id": row[3],
+ "sequence_id": row[2],
+ "position": row[0],
+ "state": _state,
+ "created_at": row[4],
+ "status": row[5],
+ }
+
+ async def save(
+ self,
+ partition_key: str,
+ app_id: str,
+ sequence_id: int,
+ position: str,
+ state: state.State,
+ status: Literal["completed", "failed"],
+ **kwargs,
+ ):
+ """
+ Saves the state for a given app_id, sequence_id, and position.
+
+ This method connects to the SQLite database, converts the state to a JSON string, and inserts a new record
+ into the table with the provided partition_key, app_id, sequence_id, position, and state. After the operation,
+ it commits the changes and closes the connection to the database.
+
+ :param partition_key: The partition key. This could be None, but it's up to the persistor to whether
+ that is a valid value it can handle.
+ :param app_id: The identifier for the app instance being recorded.
+ :param sequence_id: The state corresponding to a specific point in time.
+ :param position: The position in the sequence of states.
+ :param state: The state to be saved, an instance of the State class.
+ :param status: The status of this state, either "completed" or "failed". If "failed" the state is what it was
+ before the action was applied.
+ :return: None
+ """
+ logger.debug(
+ "saving %s, %s, %s, %s, %s, %s",
+ partition_key,
+ app_id,
+ sequence_id,
+ position,
+ state,
+ status,
+ )
+
+ json_state = json.dumps(state.serialize(**self.serde_kwargs))
+ query = (
+ f"INSERT INTO {self.table_name} (partition_key, app_id, sequence_id, position, state, status) "
+ "VALUES ($1, $2, $3, $4, $5, $6)"
+ )
+ await self.connection.execute(
+ query, partition_key, app_id, sequence_id, position, json_state, status
+ )
+
+ async def cleanup(self):
+ """Closes the connection to the database."""
+ await self.connection.close()
diff --git a/burr/integrations/persisters/b_psycopg2.py b/burr/integrations/persisters/b_psycopg2.py
new file mode 100644
index 00000000..245b42a3
--- /dev/null
+++ b/burr/integrations/persisters/b_psycopg2.py
@@ -0,0 +1,290 @@
+from burr.integrations import base
+
+try:
+ import psycopg2
+except ImportError as e:
+ base.require_plugin(e, "postgresql")
+
+import json
+import logging
+from typing import Literal, Optional
+
+from burr.core import persistence, state
+
+logger = logging.getLogger(__name__)
+
+
+class PostgreSQLPersister(persistence.BaseStatePersister):
+ """Class for PostgreSQL persistence of state. This is a simple implementation.
+
+ To try it out locally with docker -- here's a command -- change the values as appropriate.
+
+ .. code:: bash
+
+ docker run --name local-psql \ # container name
+ -v local_psql_data:/SOME/FILE_PATH/ \ # mounting a volume for data persistence
+ -p 54320:5432 \ # port mapping
+ -e POSTGRES_PASSWORD=my_password \ # superuser password
+ -d postgres # database name
+
+ Then you should be able to create the class like this:
+
+ .. code:: python
+
+ p = PostgreSQLPersister.from_values("postgres", "postgres", "my_password",
+ "localhost", 54320, table_name="burr_state")
+
+
+ """
+
+ PARTITION_KEY_DEFAULT = ""
+
+ @classmethod
+ def from_config(cls, config: dict) -> "PostgreSQLPersister":
+ """Creates a new instance of the PostgreSQLPersister from a configuration dictionary."""
+ return cls.from_values(**config)
+
+ @classmethod
+ def from_values(
+ cls,
+ db_name: str,
+ user: str,
+ password: str,
+ host: str,
+ port: int,
+ table_name: str = "burr_state",
+ ):
+ """Builds a new instance of the PostgreSQLPersister from the provided values.
+
+ :param db_name: the name of the PostgreSQL database.
+ :param user: the username to connect to the PostgreSQL database.
+ :param password: the password to connect to the PostgreSQL database.
+ :param host: the host of the PostgreSQL database.
+ :param port: the port of the PostgreSQL database.
+ :param table_name: the table name to store things under.
+ """
+ connection = psycopg2.connect(
+ dbname=db_name, user=user, password=password, host=host, port=port
+ )
+ return cls(connection, table_name)
+
+ def __init__(self, connection, table_name: str = "burr_state", serde_kwargs: dict = None):
+ """Constructor
+
+ :param connection: the connection to the PostgreSQL database.
+ :param table_name: the table name to store things under.
+ """
+ self.table_name = table_name
+ self.connection = connection
+ self.serde_kwargs = serde_kwargs or {}
+ self._initialized = False
+
+ def __enter__(self):
+ return self
+
+ def __exit__(self, exc_type, exc_value, traceback):
+ self.connection.close()
+ return False
+
+ def set_serde_kwargs(self, serde_kwargs: dict):
+ """Sets the serde_kwargs for the persister."""
+ self.serde_kwargs = serde_kwargs
+
+ def create_table(self, table_name: str):
+ """Helper function to create the table where things are stored."""
+ cursor = self.connection.cursor()
+ cursor.execute(
+ f"""
+ CREATE TABLE IF NOT EXISTS {table_name} (
+ partition_key TEXT DEFAULT '{self.PARTITION_KEY_DEFAULT}',
+ app_id TEXT NOT NULL,
+ sequence_id INTEGER NOT NULL,
+ position TEXT NOT NULL,
+ status TEXT NOT NULL,
+ state JSONB NOT NULL,
+ created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
+ PRIMARY KEY (partition_key, app_id, sequence_id, position)
+ )"""
+ )
+ cursor.execute(
+ f"""
+ CREATE INDEX IF NOT EXISTS {table_name}_created_at_index ON {table_name} (created_at);
+ """
+ )
+ self.connection.commit()
+
+ def initialize(self):
+ """Creates the table"""
+ self.create_table(self.table_name)
+ self._initialized = True
+
+ def is_initialized(self) -> bool:
+ """This checks to see if the table has been created in the database or not.
+ It defaults to using the initialized field, else queries the database to see if the table exists.
+ It then sets the initialized field to True if the table exists.
+ """
+ if self._initialized:
+ return True
+ cursor = self.connection.cursor()
+ cursor.execute(
+ "SELECT EXISTS (SELECT FROM information_schema.tables WHERE table_name = %s)",
+ (self.table_name,),
+ )
+ self._initialized = cursor.fetchone()[0]
+ return self._initialized
+
+ def list_app_ids(self, partition_key: str, **kwargs) -> list[str]:
+ """Lists the app_ids for a given partition_key."""
+ cursor = self.connection.cursor()
+ cursor.execute(
+ f"SELECT DISTINCT app_id, created_at FROM {self.table_name} "
+ "WHERE partition_key = %s "
+ "ORDER BY created_at DESC",
+ (partition_key,),
+ )
+ app_ids = [row[0] for row in cursor.fetchall()]
+ return app_ids
+
+ def load(
+ self, partition_key: Optional[str], app_id: str, sequence_id: int = None, **kwargs
+ ) -> Optional[persistence.PersistedStateData]:
+ """Loads state for a given partition id.
+
+ Depending on the parameters, this will return the last thing written, the last thing written for a given app_id,
+ or a specific sequence_id for a given app_id.
+
+ :param partition_key:
+ :param app_id:
+ :param sequence_id:
+ :return:
+ """
+ if partition_key is None:
+ partition_key = self.PARTITION_KEY_DEFAULT
+ logger.debug("Loading %s, %s, %s", partition_key, app_id, sequence_id)
+ cursor = self.connection.cursor()
+ if app_id is None:
+ # get latest for all app_ids
+ cursor.execute(
+ f"SELECT position, state, sequence_id, app_id, created_at, status FROM {self.table_name} "
+ f"WHERE partition_key = %s "
+ f"ORDER BY CREATED_AT DESC LIMIT 1",
+ (partition_key,),
+ )
+ elif sequence_id is None:
+ cursor.execute(
+ f"SELECT position, state, sequence_id, app_id, created_at, status FROM {self.table_name} "
+ f"WHERE partition_key = %s AND app_id = %s "
+ f"ORDER BY sequence_id DESC LIMIT 1",
+ (partition_key, app_id),
+ )
+ else:
+ cursor.execute(
+ f"SELECT position, state, sequence_id, app_id, created_at, status FROM {self.table_name} "
+ f"WHERE partition_key = %s AND app_id = %s AND sequence_id = %s ",
+ (partition_key, app_id, sequence_id),
+ )
+ row = cursor.fetchone()
+ if row is None:
+ return None
+ _state = state.State.deserialize(row[1], **self.serde_kwargs)
+ return {
+ "partition_key": partition_key,
+ "app_id": row[3],
+ "sequence_id": row[2],
+ "position": row[0],
+ "state": _state,
+ "created_at": row[4],
+ "status": row[5],
+ }
+
+ def save(
+ self,
+ partition_key: str,
+ app_id: str,
+ sequence_id: int,
+ position: str,
+ state: state.State,
+ status: Literal["completed", "failed"],
+ **kwargs,
+ ):
+ """
+ Saves the state for a given app_id, sequence_id, and position.
+
+ This method connects to the SQLite database, converts the state to a JSON string, and inserts a new record
+ into the table with the provided partition_key, app_id, sequence_id, position, and state. After the operation,
+ it commits the changes and closes the connection to the database.
+
+ :param partition_key: The partition key. This could be None, but it's up to the persistor to whether
+ that is a valid value it can handle.
+ :param app_id: The identifier for the app instance being recorded.
+ :param sequence_id: The state corresponding to a specific point in time.
+ :param position: The position in the sequence of states.
+ :param state: The state to be saved, an instance of the State class.
+ :param status: The status of this state, either "completed" or "failed". If "failed" the state is what it was
+ before the action was applied.
+ :return: None
+ """
+ logger.debug(
+ "saving %s, %s, %s, %s, %s, %s",
+ partition_key,
+ app_id,
+ sequence_id,
+ position,
+ state,
+ status,
+ )
+ cursor = self.connection.cursor()
+ json_state = json.dumps(state.serialize(**self.serde_kwargs))
+ cursor.execute(
+ f"INSERT INTO {self.table_name} (partition_key, app_id, sequence_id, position, state, status) "
+ "VALUES (%s, %s, %s, %s, %s, %s)",
+ (partition_key, app_id, sequence_id, position, json_state, status),
+ )
+ self.connection.commit()
+
+ def cleanup(self):
+ """Closes the connection to the database."""
+ self.connection.close()
+
+ def __del__(self):
+ # This should be deprecated -- using __del__ is unreliable for closing connections to db's;
+ # the preferred way should be for the user to use a context manager or use the `.cleanup()`
+ # method within a REST API framework.
+
+ # closes connection at end when things are being shutdown.
+ self.connection.close()
+
+ def __getstate__(self) -> dict:
+ state = self.__dict__.copy()
+ if not hasattr(self.connection, "info"):
+ logger.warning(
+ "Postgresql information for connection object not available. Cannot serialize persister."
+ )
+ return state
+ state["connection_params"] = {
+ "dbname": self.connection.info.dbname,
+ "user": self.connection.info.user,
+ "password": self.connection.info.password,
+ "host": self.connection.info.host,
+ "port": self.connection.info.port,
+ }
+ del state["connection"]
+ return state
+
+ def __setstate__(self, state: dict):
+ connection_params = state.pop("connection_params")
+ # we assume normal psycopg2 client.
+ self.connection = psycopg2.connect(**connection_params)
+ self.__dict__.update(state)
+
+
+if __name__ == "__main__":
+ # test the PostgreSQLPersister class
+ persister = PostgreSQLPersister.from_values(
+ "postgres", "postgres", "my_password", "localhost", 54320, table_name="burr_state"
+ )
+
+ persister.initialize()
+ persister.save("pk", "app_id", 1, "pos", state.State({"a": 1, "b": 2}), "completed")
+ print(persister.list_app_ids("pk"))
+ print(persister.load("pk", "app_id"))
diff --git a/burr/integrations/persisters/postgresql.py b/burr/integrations/persisters/postgresql.py
index 86f50f66..15bdaa79 100644
--- a/burr/integrations/persisters/postgresql.py
+++ b/burr/integrations/persisters/postgresql.py
@@ -1,56 +1,31 @@
+"""This module is depricated. Please import from b_psycopg2.py."""
+
from burr.integrations import base
+from burr.integrations.persisters.b_psycopg2 import PostgreSQLPersister as Psycopg2Persister
try:
import psycopg2
except ImportError as e:
base.require_plugin(e, "postgresql")
-import json
+
import logging
-from typing import Literal, Optional
-from burr.core import persistence, state
+from burr.core import state
logger = logging.getLogger(__name__)
+logger.warning(
+ "This class is deprecated and has been moved. "
+ "Please import PostgreSQLPersister from b_psycopg2.py."
+)
-class PostgreSQLPersister(persistence.BaseStatePersister):
- """Class for PostgreSQL persistence of state. This is a simple implementation.
-
- To try it out locally with docker -- here's a command -- change the values as appropriate.
-
- .. code:: bash
-
- docker run --name local-psql \ # container name
- -v local_psql_data:/SOME/FILE_PATH/ \ # mounting a volume for data persistence
- -p 54320:5432 \ # port mapping
- -e POSTGRES_PASSWORD=my_password \ # superuser password
- -d postgres # database name
-
- Then you should be able to create the class like this:
-
- .. code:: python
-
- p = PostgreSQLPersister.from_values("postgres", "postgres", "my_password",
- "localhost", 54320, table_name="burr_state")
-
+class PostgreSQLPersister(Psycopg2Persister):
+ """A class used to represent the Postgresql Persister.
+ This class is deprecated and has been moved to b_psycopg2.py.
"""
- PARTITION_KEY_DEFAULT = ""
-
- @classmethod
- def from_config(cls, config: dict) -> "PostgreSQLPersister":
- """Creates a new instance of the PostgreSQLPersister from a configuration dictionary."""
- return cls.from_values(
- db_name=config["db_name"],
- user=config["user"],
- password=config["password"],
- host=config["host"],
- port=config["port"],
- table_name=config.get("table_name", "burr_state"),
- )
-
@classmethod
def from_values(
cls,
@@ -73,201 +48,7 @@ class PostgreSQLPersister(persistence.BaseStatePersister):
connection = psycopg2.connect(
dbname=db_name, user=user, password=password, host=host, port=port
)
- return cls(connection, table_name)
-
- def __init__(self, connection, table_name: str = "burr_state", serde_kwargs: dict = None):
- """Constructor
-
- :param connection: the connection to the PostgreSQL database.
- :param table_name: the table name to store things under.
- """
- self.table_name = table_name
- self.connection = connection
- self.serde_kwargs = serde_kwargs or {}
- self._initialized = False
-
- def set_serde_kwargs(self, serde_kwargs: dict):
- """Sets the serde_kwargs for the persister."""
- self.serde_kwargs = serde_kwargs
-
- def create_table(self, table_name: str):
- """Helper function to create the table where things are stored."""
- cursor = self.connection.cursor()
- cursor.execute(
- f"""
- CREATE TABLE IF NOT EXISTS {table_name} (
- partition_key TEXT DEFAULT '{self.PARTITION_KEY_DEFAULT}',
- app_id TEXT NOT NULL,
- sequence_id INTEGER NOT NULL,
- position TEXT NOT NULL,
- status TEXT NOT NULL,
- state JSONB NOT NULL,
- created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
- PRIMARY KEY (partition_key, app_id, sequence_id, position)
- )"""
- )
- cursor.execute(
- f"""
- CREATE INDEX IF NOT EXISTS {table_name}_created_at_index ON {table_name} (created_at);
- """
- )
- self.connection.commit()
-
- def initialize(self):
- """Creates the table"""
- self.create_table(self.table_name)
- self._initialized = True
-
- def is_initialized(self) -> bool:
- """This checks to see if the table has been created in the database or not.
- It defaults to using the initialized field, else queries the database to see if the table exists.
- It then sets the initialized field to True if the table exists.
- """
- if self._initialized:
- return True
- cursor = self.connection.cursor()
- cursor.execute(
- "SELECT EXISTS (SELECT FROM information_schema.tables WHERE table_name = %s)",
- (self.table_name,),
- )
- self._initialized = cursor.fetchone()[0]
- return self._initialized
-
- def list_app_ids(self, partition_key: str, **kwargs) -> list[str]:
- """Lists the app_ids for a given partition_key."""
- cursor = self.connection.cursor()
- cursor.execute(
- f"SELECT DISTINCT app_id, created_at FROM {self.table_name} "
- "WHERE partition_key = %s "
- "ORDER BY created_at DESC",
- (partition_key,),
- )
- app_ids = [row[0] for row in cursor.fetchall()]
- return app_ids
-
- def load(
- self, partition_key: Optional[str], app_id: str, sequence_id: int = None, **kwargs
- ) -> Optional[persistence.PersistedStateData]:
- """Loads state for a given partition id.
-
- Depending on the parameters, this will return the last thing written, the last thing written for a given app_id,
- or a specific sequence_id for a given app_id.
-
- :param partition_key:
- :param app_id:
- :param sequence_id:
- :return:
- """
- if partition_key is None:
- partition_key = self.PARTITION_KEY_DEFAULT
- logger.debug("Loading %s, %s, %s", partition_key, app_id, sequence_id)
- cursor = self.connection.cursor()
- if app_id is None:
- # get latest for all app_ids
- cursor.execute(
- f"SELECT position, state, sequence_id, app_id, created_at, status FROM {self.table_name} "
- f"WHERE partition_key = %s "
- f"ORDER BY CREATED_AT DESC LIMIT 1",
- (partition_key,),
- )
- elif sequence_id is None:
- cursor.execute(
- f"SELECT position, state, sequence_id, app_id, created_at, status FROM {self.table_name} "
- f"WHERE partition_key = %s AND app_id = %s "
- f"ORDER BY sequence_id DESC LIMIT 1",
- (partition_key, app_id),
- )
- else:
- cursor.execute(
- f"SELECT position, state, sequence_id, app_id, created_at, status FROM {self.table_name} "
- f"WHERE partition_key = %s AND app_id = %s AND sequence_id = %s ",
- (partition_key, app_id, sequence_id),
- )
- row = cursor.fetchone()
- if row is None:
- return None
- _state = state.State.deserialize(row[1], **self.serde_kwargs)
- return {
- "partition_key": partition_key,
- "app_id": row[3],
- "sequence_id": row[2],
- "position": row[0],
- "state": _state,
- "created_at": row[4],
- "status": row[5],
- }
-
- def save(
- self,
- partition_key: str,
- app_id: str,
- sequence_id: int,
- position: str,
- state: state.State,
- status: Literal["completed", "failed"],
- **kwargs,
- ):
- """
- Saves the state for a given app_id, sequence_id, and position.
-
- This method connects to the SQLite database, converts the state to a JSON string, and inserts a new record
- into the table with the provided partition_key, app_id, sequence_id, position, and state. After the operation,
- it commits the changes and closes the connection to the database.
-
- :param partition_key: The partition key. This could be None, but it's up to the persistor to whether
- that is a valid value it can handle.
- :param app_id: The identifier for the app instance being recorded.
- :param sequence_id: The state corresponding to a specific point in time.
- :param position: The position in the sequence of states.
- :param state: The state to be saved, an instance of the State class.
- :param status: The status of this state, either "completed" or "failed". If "failed" the state is what it was
- before the action was applied.
- :return: None
- """
- logger.debug(
- "saving %s, %s, %s, %s, %s, %s",
- partition_key,
- app_id,
- sequence_id,
- position,
- state,
- status,
- )
- cursor = self.connection.cursor()
- json_state = json.dumps(state.serialize(**self.serde_kwargs))
- cursor.execute(
- f"INSERT INTO {self.table_name} (partition_key, app_id, sequence_id, position, state, status) "
- "VALUES (%s, %s, %s, %s, %s, %s)",
- (partition_key, app_id, sequence_id, position, json_state, status),
- )
- self.connection.commit()
-
- def __del__(self):
- # closes connection at end when things are being shutdown.
- self.connection.close()
-
- def __getstate__(self) -> dict:
- state = self.__dict__.copy()
- if not hasattr(self.connection, "info"):
- logger.warning(
- "Postgresql information for connection object not available. Cannot serialize persister."
- )
- return state
- state["connection_params"] = {
- "dbname": self.connection.info.dbname,
- "user": self.connection.info.user,
- "password": self.connection.info.password,
- "host": self.connection.info.host,
- "port": self.connection.info.port,
- }
- del state["connection"]
- return state
-
- def __setstate__(self, state: dict):
- connection_params = state.pop("connection_params")
- # we assume normal psycopg2 client.
- self.connection = psycopg2.connect(**connection_params)
- self.__dict__.update(state)
+ return Psycopg2Persister(connection, table_name)
if __name__ == "__main__":
diff --git a/docs/reference/persister.rst b/docs/reference/persister.rst
index e6068023..ebc92515 100644
--- a/docs/reference/persister.rst
+++ b/docs/reference/persister.rst
@@ -98,3 +98,8 @@ Currently we support the following, although we highly recommend you contribute
:members:
.. automethod:: __init__
+
+.. autoclass:: burr.integrations.persisters.postgresql.AsyncPostgreSQLPersister
+ :members:
+
+ .. automethod:: __init__
diff --git a/pyproject.toml b/pyproject.toml
index 6a76a202..b982bce3 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -45,7 +45,8 @@ aiosqlite = [
]
postgresql = [
- "psycopg2-binary"
+ "psycopg2-binary",
+ "asyncpg"
]
redis = [
@@ -62,6 +63,7 @@ tests = [
"langchain_community",
"pandas",
"psycopg2",
+ "asyncpg",
"pydantic[email]",
"pyarrow",
"redis",
diff --git a/tests/integrations/persisters/test_postgresql.py b/tests/integrations/persisters/test_postgresql.py
index ea4f0794..81ded15e 100644
--- a/tests/integrations/persisters/test_postgresql.py
+++ b/tests/integrations/persisters/test_postgresql.py
@@ -4,7 +4,8 @@ import pickle
import pytest
from burr.core import state
-from burr.integrations.persisters.postgresql import PostgreSQLPersister
+from burr.integrations.persisters.b_asyncpg import AsyncPostgreSQLPersister
+from burr.integrations.persisters.b_psycopg2 import PostgreSQLPersister
if not os.environ.get("BURR_CI_INTEGRATION_TESTS") == "true":
pytest.skip("Skipping integration tests", allow_module_level=True)
@@ -22,7 +23,7 @@ def postgresql_persister():
)
persister.initialize()
yield persister
- persister.connection.close()
+ persister.cleanup()
def test_save_and_load_state(postgresql_persister):
@@ -86,3 +87,60 @@ def test_serialization_with_pickle(postgresql_persister):
data = deserialized_persister.load("pk", "app_id_serde", 1)
assert data["state"].get_all() == {"a": 1, "b": 2}
+
+
+async def asyncpostgresql_persister():
+ persister = await AsyncPostgreSQLPersister.from_values(
+ db_name="postgres",
+ user="postgres",
+ password="postgres",
+ host="localhost",
+ port=5432,
+ table_name="testtable_async",
+ )
+ await persister.initialize()
+ yield persister
+ await persister.cleanup()
+
+
+async def test_async_pg_fixture(asyncpostgresql_persister):
+ assert await asyncpostgresql_persister.is_initialized()
+
+
+async def test_async_is_initialized_false():
+ persister = await AsyncPostgreSQLPersister.from_values(
+ db_name="postgres",
+ user="postgres",
+ password="postgres",
+ host="localhost",
+ port=5432,
+ table_name="testtable_async2",
+ )
+ assert not await persister.is_initialized()
+
+
+async def test_async_save_and_load_state(asyncpostgresql_persister):
+ await asyncpostgresql_persister.save(
+ "pk", "app_id", 1, "pos", state.State({"a": 1, "b": 2}), "completed"
+ )
+ data = await asyncpostgresql_persister.load("pk", "app_id", 1)
+ print(data)
+ assert data["state"].get_all() == {"a": 1, "b": 2}
+
+
+async def test_async_list_app_ids(asyncpostgresql_persister):
+ await asyncpostgresql_persister.save(
+ "pk", "app_id1", 1, "pos1", state.State({"a": 1}), "completed"
+ )
+ await asyncpostgresql_persister.save(
+ "pk", "app_id2", 2, "pos2", state.State({"b": 2}), "completed"
+ )
+ app_ids = await asyncpostgresql_persister.list_app_ids("pk")
+ assert "app_id1" in app_ids
+ assert "app_id2" in app_ids
+
+
+async def test_async_load_nonexistent_key(asyncpostgresql_persister):
+ state_data = await asyncpostgresql_persister.load("pk", "nonexistent_key")
+ assert state_data is None