Build applications that make decisions (chatbots, agents, simulations, etc...). Monitor, trace, persist, and execute on your own infrastructure.
Python aiburrchatbot-frameworkdagsgenerative-aigraphshacktoberfestllmopsllmsmlopspersistent-data-structurestate-machinestate-managementvisibility
Moves aiosqlite/persister import to plugins
elijahbenizzy9c7fbb83d343
diff --git a/burr/core/persistence.py b/burr/core/persistence.pyindex d6574540..3b302ff1 100644--- a/burr/core/persistence.py+++ b/burr/core/persistence.py@@ -6,8 +6,6 @@ from abc import ABCMetafrom collections import defaultdictfrom typing import Any, Dict, Literal, Optional, TypedDict-import aiosqlite-from burr.common.types import BaseCopyablefrom burr.core import Actionfrom burr.core.state import State, logger@@ -502,243 +500,6 @@ class SQLitePersister(BaseStatePersister, BaseCopyable):)-class AsyncSQLitePersister(AsyncBaseStatePersister, BaseCopyable):- """Class for asynchronous SQLite persistence of state. This is a simple implementation.-- SQLite is specifically single-threaded and `aiosqlite <https://aiosqlite.omnilib.dev/en/latest/index.html>`_- creates async support through multi-threading. This persister is mainly here for quick prototyping and testing;- we suggest to consider a different database with native async support for production.-- Note the third-party library `aiosqlite <https://aiosqlite.omnilib.dev/en/latest/index.html>`_,- is maintained and considered stable considered stable: https://github.com/omnilib/aiosqlite/issues/309.- """-- def copy(self) -> "Self":- return AsyncSQLitePersister(- db_path=self.db_path,- table_name=self.table_name,- serde_kwargs=self.serde_kwargs,- connect_kwargs=self._connect_kwargs,- )-- PARTITION_KEY_DEFAULT = ""-- @classmethod- async def from_values(- cls,- db_path: str,- table_name: str = "burr_state",- serde_kwargs: dict = None,- connect_kwargs: dict = None,- ) -> "AsyncSQLitePersister":- """Creates a new instance of the AsyncSQLitePersister from passed in values.-- :param db_path: the path the DB will be stored.- :param table_name: the table name to store things under.- :param serde_kwargs: kwargs for state serialization/deserialization.- :param connect_kwargs: kwargs to pass to the aiosqlite.connect method.- :return: async sqlite persister instance with an open connection. You are responsible- for closing the connection yourself.- """- connection = await aiosqlite.connect(- db_path, **connect_kwargs if connect_kwargs is not None else {}- )- return cls(connection, table_name, serde_kwargs)-- def __init__(- self,- connection,- table_name: str = "burr_state",- serde_kwargs: dict = None,- ):- """Constructor.-- NOTE: you are responsible to handle closing of the connection / teardown manually. To help,- we provide a close() method.-- :param connection: the path the DB will be stored.- :param table_name: the table name to store things under.- :param serde_kwargs: kwargs for state serialization/deserialization.- """- self.connection = connection- self.table_name = table_name- self.serde_kwargs = serde_kwargs or {}- self._initialized = False-- async def create_table_if_not_exists(self, table_name: str):- """Helper function to create the table where things are stored if it doesn't exist."""- cursor = await self.connection.cursor()- await cursor.execute(- f"""- CREATE TABLE IF NOT EXISTS {table_name} (- partition_key TEXT DEFAULT '{AsyncSQLitePersister.PARTITION_KEY_DEFAULT}',- app_id TEXT NOT NULL,- sequence_id INTEGER NOT NULL,- position TEXT NOT NULL,- status TEXT NOT NULL,- state TEXT NOT NULL,- created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,- PRIMARY KEY (partition_key, app_id, sequence_id, position)- )"""- )- await cursor.execute(- f"""- CREATE INDEX IF NOT EXISTS {table_name}_created_at_index ON {table_name} (created_at);- """- )- await self.connection.commit()-- async def initialize(self):- """Asynchronously creates the table if it doesn't exist"""- # Usage- await self.create_table_if_not_exists(self.table_name)- self._initialized = True-- async def is_initialized(self) -> bool:- """This checks to see if the table has been created in the database or not.- It defaults to using the initialized field, else queries the database to see if the table exists.- It then sets the initialized field to True if the table exists.- """- if self._initialized:- return True-- cursor = await self.connection.cursor()- await cursor.execute(- "SELECT name FROM sqlite_master WHERE type='table' AND name=?", (self.table_name,)- )- self._initialized = await cursor.fetchone() is not None- return self._initialized-- async def list_app_ids(self, partition_key: Optional[str] = None, **kwargs) -> list[str]:- partition_key = (- partition_key- if partition_key is not None- else AsyncSQLitePersister.PARTITION_KEY_DEFAULT- )-- cursor = await self.connection.cursor()- await cursor.execute(- f"SELECT DISTINCT app_id FROM {self.table_name} "- f"WHERE partition_key = ? "- f"ORDER BY created_at DESC",- (partition_key,),- )- app_ids = [row[0] for row in await cursor.fetchall()]- return app_ids-- async def load(- self,- partition_key: Optional[str],- app_id: Optional[str],- sequence_id: Optional[int] = None,- **kwargs,- ) -> Optional[PersistedStateData]:- """Asynchronously loads state for a given partition id.-- Depending on the parameters, this will return the last thing written, the last thing written for a given app_id,- or a specific sequence_id for a given app_id.-- :param partition_key:- :param app_id:- :param sequence_id:- :return:- """- partition_key = (- partition_key- if partition_key is not None- else AsyncSQLitePersister.PARTITION_KEY_DEFAULT- )- logger.debug("Loading %s, %s, %s", partition_key, app_id, sequence_id)- cursor = await self.connection.cursor()- if app_id is None:- # get latest for all app_ids- await cursor.execute(- f"SELECT position, state, sequence_id, app_id, created_at, status FROM {self.table_name} "- f"WHERE partition_key = ? "- f"ORDER BY CREATED_AT DESC LIMIT 1",- (partition_key,),- )- elif sequence_id is None:- await cursor.execute(- f"SELECT position, state, sequence_id, app_id, created_at, status FROM {self.table_name} "- f"WHERE partition_key = ? AND app_id = ? "- f"ORDER BY sequence_id DESC LIMIT 1",- (partition_key, app_id),- )- else:- await cursor.execute(- f"SELECT position, state, sequence_id, app_id, created_at, status FROM {self.table_name} "- f"WHERE partition_key = ? AND app_id = ? AND sequence_id = ?",- (partition_key, app_id, sequence_id),- )- row = await cursor.fetchone()- if row is None:- return None- _state = State.deserialize(json.loads(row[1]), **self.serde_kwargs)- return {- "partition_key": partition_key,- "app_id": row[3],- "sequence_id": row[2],- "position": row[0],- "state": _state,- "created_at": row[4],- "status": row[5],- }-- async def save(- self,- partition_key: Optional[str],- app_id: str,- sequence_id: int,- position: str,- state: State,- status: Literal["completed", "failed"],- **kwargs,- ):- """- Asynchronously saves the state for a given app_id, sequence_id, and position.-- This method connects to the SQLite database, converts the state to a JSON string, and inserts a new record- into the table with the provided partition_key, app_id, sequence_id, position, and state. After the operation,- it commits the changes and closes the connection to the database.-- :param partition_key: The partition key. This could be None, but it's up to the persister to whether- that is a valid value it can handle.- :param app_id: The identifier for the app instance being recorded.- :param sequence_id: The state corresponding to a specific point in time.- :param position: The position in the sequence of states.- :param state: The state to be saved, an instance of the State class.- :param status: The status of this state, either "completed" or "failed". If "failed" the state is what it was- before the action was applied.- :return: None- """- logger.debug(- "saving %s, %s, %s, %s, %s, %s",- partition_key,- app_id,- sequence_id,- position,- state,- status,- )- partition_key = (- partition_key- if partition_key is not None- else AsyncSQLitePersister.PARTITION_KEY_DEFAULT- )- cursor = await self.connection.cursor()- json_state = json.dumps(state.serialize(**self.serde_kwargs))- await cursor.execute(- f"INSERT INTO {self.table_name} (partition_key, app_id, sequence_id, position, state, status) "- f"VALUES (?, ?, ?, ?, ?, ?)",- (partition_key, app_id, sequence_id, position, json_state, status),- )- await self.connection.commit()-- async def close(self):- await self.connection.close()--class InMemoryPersister(BaseStatePersister):"""In-memory persister for testing purposes. This is not recommended for production use."""@@ -846,7 +607,6 @@ class AsyncInMemoryPersister(AsyncBaseStatePersister):SQLLitePersister = SQLitePersister-AsyncSQLLitePersister = AsyncSQLitePersisterif __name__ == "__main__":s = SQLitePersister(db_path=".SQLite.db", table_name="test1")diff --git a/burr/integrations/persisters/b_aiosqlite.py b/burr/integrations/persisters/b_aiosqlite.pynew file mode 100644index 00000000..68730b5d--- /dev/null+++ b/burr/integrations/persisters/b_aiosqlite.py@@ -0,0 +1,253 @@+import json+import logging+from typing import Literal, Optional++import aiosqlite++from burr.common.types import BaseCopyable+from burr.core import State+from burr.core.persistence import AsyncBaseStatePersister, PersistedStateData++logger = logging.getLogger()++try:+ from typing import Self+except ImportError:+ Self = None+++class AsyncSQLitePersister(AsyncBaseStatePersister, BaseCopyable):+ """Class for asynchronous SQLite persistence of state. This is a simple implementation.++ SQLite is specifically single-threaded and `aiosqlite <https://aiosqlite.omnilib.dev/en/latest/index.html>`_+ creates async support through multi-threading. This persister is mainly here for quick prototyping and testing;+ we suggest to consider a different database with native async support for production.++ Note the third-party library `aiosqlite <https://aiosqlite.omnilib.dev/en/latest/index.html>`_,+ is maintained and considered stable considered stable: https://github.com/omnilib/aiosqlite/issues/309.+ """++ def copy(self) -> "Self":+ return AsyncSQLitePersister(+ db_path=self.db_path,+ table_name=self.table_name,+ serde_kwargs=self.serde_kwargs,+ connect_kwargs=self._connect_kwargs,+ )++ PARTITION_KEY_DEFAULT = ""++ @classmethod+ async def from_values(+ cls,+ db_path: str,+ table_name: str = "burr_state",+ serde_kwargs: dict = None,+ connect_kwargs: dict = None,+ ) -> "AsyncSQLitePersister":+ """Creates a new instance of the AsyncSQLitePersister from passed in values.++ :param db_path: the path the DB will be stored.+ :param table_name: the table name to store things under.+ :param serde_kwargs: kwargs for state serialization/deserialization.+ :param connect_kwargs: kwargs to pass to the aiosqlite.connect method.+ :return: async sqlite persister instance with an open connection. You are responsible+ for closing the connection yourself.+ """+ connection = await aiosqlite.connect(+ db_path, **connect_kwargs if connect_kwargs is not None else {}+ )+ return cls(connection, table_name, serde_kwargs)++ def __init__(+ self,+ connection,+ table_name: str = "burr_state",+ serde_kwargs: dict = None,+ ):+ """Constructor.++ NOTE: you are responsible to handle closing of the connection / teardown manually. To help,+ we provide a close() method.++ :param connection: the path the DB will be stored.+ :param table_name: the table name to store things under.+ :param serde_kwargs: kwargs for state serialization/deserialization.+ """+ self.connection = connection+ self.table_name = table_name+ self.serde_kwargs = serde_kwargs or {}+ self._initialized = False++ async def create_table_if_not_exists(self, table_name: str):+ """Helper function to create the table where things are stored if it doesn't exist."""+ cursor = await self.connection.cursor()+ await cursor.execute(+ f"""+ CREATE TABLE IF NOT EXISTS {table_name} (+ partition_key TEXT DEFAULT '{AsyncSQLitePersister.PARTITION_KEY_DEFAULT}',+ app_id TEXT NOT NULL,+ sequence_id INTEGER NOT NULL,+ position TEXT NOT NULL,+ status TEXT NOT NULL,+ state TEXT NOT NULL,+ created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,+ PRIMARY KEY (partition_key, app_id, sequence_id, position)+ )"""+ )+ await cursor.execute(+ f"""+ CREATE INDEX IF NOT EXISTS {table_name}_created_at_index ON {table_name} (created_at);+ """+ )+ await self.connection.commit()++ async def initialize(self):+ """Asynchronously creates the table if it doesn't exist"""+ # Usage+ await self.create_table_if_not_exists(self.table_name)+ self._initialized = True++ async def is_initialized(self) -> bool:+ """This checks to see if the table has been created in the database or not.+ It defaults to using the initialized field, else queries the database to see if the table exists.+ It then sets the initialized field to True if the table exists.+ """+ if self._initialized:+ return True++ cursor = await self.connection.cursor()+ await cursor.execute(+ "SELECT name FROM sqlite_master WHERE type='table' AND name=?", (self.table_name,)+ )+ self._initialized = await cursor.fetchone() is not None+ return self._initialized++ async def list_app_ids(self, partition_key: Optional[str] = None, **kwargs) -> list[str]:+ partition_key = (+ partition_key+ if partition_key is not None+ else AsyncSQLitePersister.PARTITION_KEY_DEFAULT+ )++ cursor = await self.connection.cursor()+ await cursor.execute(+ f"SELECT DISTINCT app_id FROM {self.table_name} "+ f"WHERE partition_key = ? "+ f"ORDER BY created_at DESC",+ (partition_key,),+ )+ app_ids = [row[0] for row in await cursor.fetchall()]+ return app_ids++ async def load(+ self,+ partition_key: Optional[str],+ app_id: Optional[str],+ sequence_id: Optional[int] = None,+ **kwargs,+ ) -> Optional[PersistedStateData]:+ """Asynchronously loads state for a given partition id.++ Depending on the parameters, this will return the last thing written, the last thing written for a given app_id,+ or a specific sequence_id for a given app_id.++ :param partition_key:+ :param app_id:+ :param sequence_id:+ :return:+ """+ partition_key = (+ partition_key+ if partition_key is not None+ else AsyncSQLitePersister.PARTITION_KEY_DEFAULT+ )+ logger.debug("Loading %s, %s, %s", partition_key, app_id, sequence_id)+ cursor = await self.connection.cursor()+ if app_id is None:+ # get latest for all app_ids+ await cursor.execute(+ f"SELECT position, state, sequence_id, app_id, created_at, status FROM {self.table_name} "+ f"WHERE partition_key = ? "+ f"ORDER BY CREATED_AT DESC LIMIT 1",+ (partition_key,),+ )+ elif sequence_id is None:+ await cursor.execute(+ f"SELECT position, state, sequence_id, app_id, created_at, status FROM {self.table_name} "+ f"WHERE partition_key = ? AND app_id = ? "+ f"ORDER BY sequence_id DESC LIMIT 1",+ (partition_key, app_id),+ )+ else:+ await cursor.execute(+ f"SELECT position, state, sequence_id, app_id, created_at, status FROM {self.table_name} "+ f"WHERE partition_key = ? AND app_id = ? AND sequence_id = ?",+ (partition_key, app_id, sequence_id),+ )+ row = await cursor.fetchone()+ if row is None:+ return None+ _state = State.deserialize(json.loads(row[1]), **self.serde_kwargs)+ return {+ "partition_key": partition_key,+ "app_id": row[3],+ "sequence_id": row[2],+ "position": row[0],+ "state": _state,+ "created_at": row[4],+ "status": row[5],+ }++ async def save(+ self,+ partition_key: Optional[str],+ app_id: str,+ sequence_id: int,+ position: str,+ state: State,+ status: Literal["completed", "failed"],+ **kwargs,+ ):+ """+ Asynchronously saves the state for a given app_id, sequence_id, and position.++ This method connects to the SQLite database, converts the state to a JSON string, and inserts a new record+ into the table with the provided partition_key, app_id, sequence_id, position, and state. After the operation,+ it commits the changes and closes the connection to the database.++ :param partition_key: The partition key. This could be None, but it's up to the persister to whether+ that is a valid value it can handle.+ :param app_id: The identifier for the app instance being recorded.+ :param sequence_id: The state corresponding to a specific point in time.+ :param position: The position in the sequence of states.+ :param state: The state to be saved, an instance of the State class.+ :param status: The status of this state, either "completed" or "failed". If "failed" the state is what it was+ before the action was applied.+ :return: None+ """+ logger.debug(+ "saving %s, %s, %s, %s, %s, %s",+ partition_key,+ app_id,+ sequence_id,+ position,+ state,+ status,+ )+ partition_key = (+ partition_key+ if partition_key is not None+ else AsyncSQLitePersister.PARTITION_KEY_DEFAULT+ )+ cursor = await self.connection.cursor()+ json_state = json.dumps(state.serialize(**self.serde_kwargs))+ await cursor.execute(+ f"INSERT INTO {self.table_name} (partition_key, app_id, sequence_id, position, state, status) "+ f"VALUES (?, ?, ?, ?, ?, ?)",+ (partition_key, app_id, sequence_id, position, json_state, status),+ )+ await self.connection.commit()++ async def close(self):+ await self.connection.close()diff --git a/docs/reference/persister.rst b/docs/reference/persister.rstindex 2c723d83..3f7ee868 100644--- a/docs/reference/persister.rst+++ b/docs/reference/persister.rst@@ -70,7 +70,7 @@ Currently we support the following, although we highly recommend you contribute.. _asyncpersistersref:-.. autoclass:: burr.core.persistence.AsyncSQLitePersister+.. autoclass:: burr.integrations.persisters.b_aiosqlite.AsyncSQLitePersister:members:.. automethod:: __init__