Build applications that make decisions (chatbots, agents, simulations, etc...). Monitor, trace, persist, and execute on your own infrastructure.
Python aiburrchatbot-frameworkdagsgenerative-aigraphshacktoberfestllmopsllmsmlopspersistent-data-structurestate-machinestate-managementvisibility
Adds postgresql persister
Stefan Krawczykafd6769d30cf
diff --git a/burr/integrations/persisters/__init__.py b/burr/integrations/persisters/__init__.pynew file mode 100644index 00000000..e69de29bdiff --git a/burr/integrations/persisters/postgresql.py b/burr/integrations/persisters/postgresql.pynew file mode 100644index 00000000..4fd154e5--- /dev/null+++ b/burr/integrations/persisters/postgresql.py@@ -0,0 +1,234 @@+try:+ import psycopg2+except ImportError:+ raise ImportError(+ "Missing plugin PostgreSQL! To use the PostgreSQL plugin, you must install the following libraries: psycopg2. "+ "You can install this with burr[postgresql] or pip install psycopg2 (replace with your package manager of choice)."+ )++import json+import logging+from typing import Literal, Optional++from burr.core import persistence, state++logger = logging.getLogger(__name__)+++class PostgreSQLPersister(persistence.BaseStatePersister):+ """Class for PostgreSQL persistence of state. This is a simple implementation.++ To try it out locally with docker -- here's a command -- change the values as appropriate.++ .. code:: bash++ docker run --name local-psql \ # container name+ -v local_psql_data:/SOME/FILE_PATH/ \ # mounting a volume for data persistence+ -p 54320:5432 \ # port mapping+ -e POSTGRES_PASSWORD=my_password \ # superuser password+ -d postgres # database name++ Then you should be able to create the class like this:++ .. code:: python++ p = PostgreSQLPersister.from_values("postgres", "postgres", "my_password",+ "localhost", 54320, table_name="burr_state")+++ """++ @classmethod+ def from_config(cls, config: dict) -> "PostgreSQLPersister":+ """Creates a new instance of the PostgreSQLPersister from a configuration dictionary."""+ return cls.from_values(+ db_name=config["db_name"],+ user=config["user"],+ password=config["password"],+ host=config["host"],+ port=config["port"],+ table_name=config.get("table_name", "burr_state"),+ )++ @classmethod+ def from_values(+ cls,+ db_name: str,+ user: str,+ password: str,+ host: str,+ port: int,+ table_name: str = "burr_state",+ ):+ """Builds a new instance of the PostgreSQLPersister from the provided values.++ :param db_name: the name of the PostgreSQL database.+ :param user: the username to connect to the PostgreSQL database.+ :param password: the password to connect to the PostgreSQL database.+ :param host: the host of the PostgreSQL database.+ :param port: the port of the PostgreSQL database.+ :param table_name: the table name to store things under.+ """+ connection = psycopg2.connect(+ dbname=db_name, user=user, password=password, host=host, port=port+ )+ return cls(connection, table_name)++ def __init__(self, connection, table_name: str = "burr_state"):+ """Constructor++ :param connection: the connection to the PostgreSQL database.+ :param table_name: the table name to store things under.+ """+ self.table_name = table_name+ self.connection = connection++ def create_table(self, table_name: str):+ """Helper function to create the table where things are stored."""+ cursor = self.connection.cursor()+ cursor.execute(+ f"""+ CREATE TABLE IF NOT EXISTS {table_name} (+ partition_key varchar(72) NOT NULL,+ app_id varchar(72) NOT NULL,+ sequence_id INTEGER NOT NULL,+ position varchar(72) NOT NULL,+ status varchar(36) NOT NULL,+ state JSONB NOT NULL,+ created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,+ PRIMARY KEY (partition_key, app_id, sequence_id, position, state)+ )"""+ )+ cursor.execute(+ f"""+ CREATE INDEX IF NOT EXISTS {table_name}_created_at_index ON {table_name} (created_at);+ """+ )+ self.connection.commit()++ def initialize(self):+ """Creates the table"""+ self.create_table(self.table_name)++ # The rest of the methods (list_app_ids, load, save, __del__) remain the same as in the SQLLitePersister class++ def list_app_ids(self, partition_key: str) -> list[str]:+ cursor = self.connection.cursor()+ cursor.execute(+ f"SELECT DISTINCT app_id, created_at FROM {self.table_name} "+ f"WHERE partition_key = %s "+ f"ORDER BY created_at DESC",+ (partition_key,),+ )+ app_ids = [row[0] for row in cursor.fetchall()]+ return app_ids++ def load(+ self, partition_key: str, app_id: str, sequence_id: int = None+ ) -> Optional[persistence.PersistedStateData]:+ """Loads state for a given partition id.++ Depending on the parameters, this will return the last thing written, the last thing written for a given app_id,+ or a specific sequence_id for a given app_id.++ :param partition_key:+ :param app_id:+ :param sequence_id:+ :return:+ """+ logger.debug("Loading %s, %s, %s", partition_key, app_id, sequence_id)+ cursor = self.connection.cursor()+ if app_id is None:+ # get latest for all app_ids+ cursor.execute(+ f"SELECT position, state, sequence_id, app_id, created_at, status FROM {self.table_name} "+ f"WHERE partition_key = %s "+ f"ORDER BY CREATED_AT DESC LIMIT 1",+ (partition_key,),+ )+ elif sequence_id is None:+ cursor.execute(+ f"SELECT position, state, sequence_id, app_id, created_at, status FROM {self.table_name} "+ f"WHERE partition_key = %s AND app_id = %s "+ f"ORDER BY sequence_id DESC LIMIT 1",+ (partition_key, app_id),+ )+ else:+ cursor.execute(+ f"SELECT position, state, seqeuence_id, app_id, created_at, status FROM {self.table_name} "+ f"WHERE partition_key = %s AND app_id = %s AND sequence_id = %s ",+ (partition_key, app_id, sequence_id),+ )+ row = cursor.fetchone()+ if row is None:+ return None+ _state = state.State(row[1]["_state"])+ return {+ "partition_key": partition_key,+ "app_id": row[3],+ "sequence_id": row[2],+ "position": row[0],+ "state": _state,+ "created_at": row[4],+ "status": row[5],+ }++ def save(+ self,+ partition_key: str,+ app_id: str,+ sequence_id: int,+ position: str,+ state: state.State,+ status: Literal["completed", "failed"],+ ):+ """+ Saves the state for a given app_id, sequence_id, and position.++ This method connects to the SQLite database, converts the state to a JSON string, and inserts a new record+ into the table with the provided partition_key, app_id, sequence_id, position, and state. After the operation,+ it commits the changes and closes the connection to the database.++ :param partition_key: The partition key. This could be None, but it's up to the persistor to whether+ that is a valid value it can handle.+ :param app_id: The identifier for the app instance being recorded.+ :param sequence_id: The state corresponding to a specific point in time.+ :param position: The position in the sequence of states.+ :param state: The state to be saved, an instance of the State class.+ :param status: The status of this state, either "completed" or "failed". If "failed" the state is what it was+ before the action was applied.+ :return: None+ """+ logger.debug(+ "saving %s, %s, %s, %s, %s, %s",+ partition_key,+ app_id,+ sequence_id,+ position,+ state,+ status,+ )+ cursor = self.connection.cursor()+ json_state = json.dumps(state.__dict__)+ cursor.execute(+ f"INSERT INTO {self.table_name} (partition_key, app_id, sequence_id, position, state, status) "+ "VALUES (%s, %s, %s, %s, %s, %s)",+ (partition_key, app_id, sequence_id, position, json_state, status),+ )+ self.connection.commit()++ def __del__(self):+ # closes connection at end when things are being shutdown.+ self.connection.close()+++if __name__ == "__main__":+ # test the PostgreSQLPersister class+ p = PostgreSQLPersister.from_values(+ "postgres", "postgres", "my_password", "localhost", 54320, table_name="burr_state"+ )++ p.initialize()+ p.save("pk", "app_id", 1, "pos", state.State({"a": 1, "b": 2}), "completed")+ print(p.list_app_ids("pk"))+ print(p.load("pk", "app_id"))diff --git a/docs/reference/persister.rst b/docs/reference/persister.rstindex 887bf057..0f0de180 100644--- a/docs/reference/persister.rst+++ b/docs/reference/persister.rst@@ -40,5 +40,12 @@ Currently we support the following, although we highly recommend you contribute.. automethod:: __init__++.. autoclass:: burr.integrations.persisters.postgresql.PostgreSQLPersister+ :members:++ .. automethod:: __init__++Note that the :py:class:`LocalTrackingClient <burr.tracking.client.LocalTrackingClient>` leverages the :py:class:`BaseStateLoader <burr.core.persistence.BaseStateLoader>` to allow loading state,although it uses different mechanisms to save state (as it tracks more than just state).diff --git a/pyproject.toml b/pyproject.tomlindex 27c81070..2cf125f6 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -37,6 +37,10 @@ graphviz = ["graphviz"]+postgresql = [+ "psycopg2-binary"+]+tests = ["pytest","pytest-asyncio",@@ -48,7 +52,8 @@ documentation = ["myst-parser","furo","sphinx-sitemap",- "sphinx-toolbox"+ "sphinx-toolbox",+ "psychopg2-binary",]tracking-client = [