Build applications that make decisions (chatbots, agents, simulations, etc...). Monitor, trace, persist, and execute on your own infrastructure.
Python aiburrchatbot-frameworkdagsgenerative-aigraphshacktoberfestllmopsllmsmlopspersistent-data-structurestate-machinestate-managementvisibility
Updates state persistence code
elijahbenizzy26eddd15b929
diff --git a/burr/core/application.py b/burr/core/application.pyindex 2eaab3f3..eaa330e2 100644--- a/burr/core/application.py+++ b/burr/core/application.py@@ -19,7 +19,9 @@ from typing import (cast,)-from burr import telemetry, visibility+from burr import telemetry+from burr import visibility+from burr.core import persistencefrom burr.core.action import (Action,Condition,@@ -32,14 +34,8 @@ from burr.core.action import (create_action,default,)-from burr.core.state import BasicStatePersistence, State-from burr.lifecycle import (- PostApplicationCreateHook,- PostEndSpanHook,- PostRunStepHook,- PreRunStepHook,- PreStartSpanHook,-)+from burr.core.persistence import BaseStateLoader, BaseStateSaver+from burr.core.state import Statefrom burr.lifecycle.base import LifecycleAdapterfrom burr.lifecycle.internal import LifecycleAdapterSet@@ -238,19 +234,18 @@ class ApplicationGraph:class Application:def __init__(- self,- actions: List[Action],- transitions: List[Transition],- state: State,- initial_step: str,- partition_key: str,- id: str,- sequence_id: int,- adapter_set: Optional[LifecycleAdapterSet] = None,+ self,+ actions: List[Action],+ transitions: List[Transition],+ state: State,+ initial_step: str,+ partition_key: Optional[str],+ uid: str,+ sequence_id: Optional[int],+ adapter_set: Optional[LifecycleAdapterSet] = None,):self._partition_key = partition_key- self._id = id- self._sequence_id = sequence_id+ self._uid = uidself._action_map = {action.name: action for action in actions}self._adjacency_map = Application._create_adjacency_map(transitions)self._transitions = transitions@@ -260,7 +255,10 @@ class Application:self._adapter_set = adapter_set if adapter_set is not None else LifecycleAdapterSet()self._graph = self._create_graph()self._adapter_set.call_all_lifecycle_hooks_sync(- "post_application_create", state=self._state, application_graph=self._graph+ "post_application_create",+ state=self._state,+ application_graph=self._graph,+ app_id=self._uid,)# TODO -- consider adding global inputs + global input factories to the builderself.dependency_factory = {@@ -268,6 +266,8 @@ class Application:visibility.tracing.TracerFactory, lifecycle_adapters=self._adapter_set)}+ if sequence_id is not None:+ self._set_sequence_id(sequence_id)# @telemetry.capture_function_usage # todo -- capture usage when we break this up into one that isn't called internally# This will be doable when we move sequence ID to the beginning of the function https://github.com/DAGWorks-Inc/burr/pull/73@@ -308,7 +308,7 @@ class Application:state=self._state,inputs=inputs,sequence_id=self.sequence_id,- app_id=self._id,+ app_id=self._uid,partition_key=self._partition_key,)exc = None@@ -331,7 +331,7 @@ class Application:if _run_hooks:self._adapter_set.call_all_lifecycle_hooks_sync("post_run_step",- app_id=self._id,+ app_id=self._uid,partition_key=self._partition_key,action=next_action,state=new_state,@@ -401,7 +401,7 @@ class Application:state=self._state,inputs=inputs,sequence_id=self.sequence_id,- app_id=self._id,+ app_id=self._uid,partition_key=self._partition_key,)exc = None@@ -438,7 +438,7 @@ class Application:result=result,sequence_id=self.sequence_id,exception=exc,- app_id=self._id,+ app_id=self._uid,partition_key=self._partition_key,)@@ -755,12 +755,12 @@ class Application:)self._validate_streaming_inputs(halt_after)+ self._increment_sequence_id()next_action = self.get_next_action()if next_action is None:raise ValueError(f"Cannot stream result -- no next action found! Prior action was: {self._state[PRIOR_STEP]}")- self._increment_sequence_id() # TODO: is this in the right place?if next_action.name not in halt_after:# fast forward until we get to the actionnext_action, results, state = self.run(@@ -779,7 +779,6 @@ class Application:sequence_id=self.sequence_id,exception=None,)- self._increment_sequence_id()return next_action, StreamingResultContainer.pass_through(results=results, final_state=state)@@ -814,7 +813,6 @@ class Application:exception=exc,)# we want to increment regardless of failure- self._increment_sequence_id()if not next_action.streaming:# In this case we are halting at a non-streaming condition@@ -828,7 +826,6 @@ class Application:sequence_id=self.sequence_id,exception=None,)- self._increment_sequence_id()return action, StreamingResultContainer.pass_through(results=result, final_state=state)@@ -1005,24 +1002,43 @@ class Application:@propertydef sequence_id(self) -> Optional[int]:"""gives the sequence ID of the current (next) action.- This is incremented after every step is taken -- meaning that incremeneting- it is the very last action that is done. Any logging, etc... will use the current+ This is incremented prior to every step. Any logging, etc... will use the currentstep's sequence ID- :return:+ :return: The sequence ID of the current (next) action"""- return self._sequence_id+ return self._state.get(SEQUENCE_ID)def _increment_sequence_id(self):- self._sequence_id = self._sequence_id + 1- self._state = self._state.update(**{SEQUENCE_ID: self.sequence_id})+ if SEQUENCE_ID not in self._state:+ self._state = self._state.update(**{SEQUENCE_ID: 0})+ else:+ self._state = self._state.update(**{SEQUENCE_ID: self.sequence_id + 1})++ def _set_sequence_id(self, sequence_id: int):+ self._state = self._state.update(**{SEQUENCE_ID: sequence_id})@property- def id(self) -> str:- return self._id+ def uid(self) -> str:+ """Unique ID for the application. This must be unique across *all* applications in a search space.+ This is used by persistence/tracking to ensure that applications have meanings.++ Every application has this -- if not assigned, it will be randomly generated.++ :return: The unique ID for the application+ """+ return self._uid@property- def partition_key(self) -> str:+ def partition_key(self) -> Optional[str]:+ """Partition key for the application. This is designed to add semantic meaning to+ the application, and be leveraged by persistence systems to select/find applications.++ Note this is optional -- if it is not included, you will need to use a persister that+ supports a null partition key.++ :return: The partition key, None if not set+ """return self._partition_key@@ -1090,15 +1106,24 @@ class ApplicationBuilder:self.start: Optional[str] = Noneself.lifecycle_adapters: List[LifecycleAdapter] = list()self.app_id: str = str(uuid.uuid4())- self.partition_key: str = None- self.sequence_id: int = 0- self.persister = None+ self.partition_key: Optional[str] = None+ self.sequence_id: Optional[int] = None+ self.initializer = Noneself.use_entrypoint_from_save_state: bool = Noneself.default_state: dict = Nonedef with_identifiers(self, app_id: str = None, partition_key: str = None, sequence_id: int = None) -> "ApplicationBuilder":+ """Assigns various identifiers to the application. This is used for tracking, persistence, etc...++ :param app_id: Application ID -- this will be assigned to a uuid if not set.+ :param partition_key: Partition key -- this is used for disambiguating groups of applications. For instance, a unique user ID, etc...+ This is coupled to persistence, and is used to query for/select application runs.+ :param sequence_id: Sequence ID that we want this to start at. If you're using ``.initialize``, this will be set. Otherwise this is+ solely for resetting/starting at a specified position.+ :return: The application builder for future chaining.+ """if app_id is not None:self.app_id = app_idif partition_key is not None:@@ -1116,6 +1141,12 @@ class ApplicationBuilder::param kwargs: Key-value pairs to set in the state:return: The application builder for future chaining."""+ if self.initializer is not None:+ raise ValueError(+ "You cannot set state if you are loading state"+ "the .initialize_from() API. Either allow the persister to set the "+ "state, or set the state manually."+ )if self.state is not None:self.state = self.state.update(**kwargs)else:@@ -1130,6 +1161,12 @@ class ApplicationBuilder::return: The application builder for future chaining."""# TODO -- validate only called once+ if self.start is not None:+ raise ValueError(+ "You cannot set the entrypoint if you are loading a persister using "+ "the .initialize_from() API. Either allow the persister to set the "+ "entrypoint/provide a default, or set the entrypoint + state manually."+ )self.start = actionreturn self@@ -1193,62 +1230,85 @@ class ApplicationBuilder:def with_tracker(self,- tracker: Union[Literal["local"], object] = "local",+ tracker: Union[+ Literal["local"], LifecycleAdapter+ ] = "local", # TODO -- tighten type-check for tracker. For now it has the lifecycle adapterproject: str = "default",params: Dict[str, Any] = None,):"""Adds a "tracker" to the application. The tracker specifiesa project name (used for disambiguating groups of tracers), and plugs into the- Burr UI. Currently the only supported tracker is local, which takes in the params- `storage_dir` and `app_id`, which have automatic defaults.+ Burr UI. This can either be:++ 1. A string (the only supported one right now is "local"), and a set of parameters for a set of supported trackers.+ 2. A lifecycle adapter object that does tracking (up to you how to implement it).++ (1) internally creates a :py:class:`LocalTrackingClient <burr.tracking.client.LocalTrackingClient>` object, and adds it to the lifecycle adapters.+ (2) adds the lifecycle adapter to the lifecycle adapters.:param tracker: Tracker to use. ``local`` creates one, else pass one in.- :param project: Project name -- used if the tracker is local.- :param params: Parameters to pass to the tracker if it's local.+ :param project: Project name -- used if the tracker is string-specified (local).+ :param params: Parameters to pass to the tracker if it's string-specified (local).:return: The application builder for future chaining."""- if params is None:- params = {}# if it's a lifecycle adapter, just add it- if isinstance(- tracker,- (- PostRunStepHook,- PreRunStepHook,- PostApplicationCreateHook,- PostEndSpanHook,- PreStartSpanHook,- ),- ):- self.lifecycle_adapters.append(tracker)- elif tracker == "local":- from burr.tracking.client import LocalTrackingClient-- kwargs = {"project": project, "app_id": self.app_id}- kwargs.update(params)- self.lifecycle_adapters.append(LocalTrackingClient(**kwargs))+ if isinstance(tracker, str):+ if params is None:+ params = {}+ if tracker == "local":+ from burr.tracking.client import LocalTrackingClient++ kwargs = {"project": project}+ kwargs.update(params)+ self.lifecycle_adapters.append(LocalTrackingClient(**kwargs))+ else:+ raise ValueError(f"Tracker {tracker}:{project} not supported")else:- raise ValueError(f"Tracker {tracker}:{project} not supported")+ self.lifecycle_adapters.append(tracker)+ if params is not None:+ raise ValueError(+ "Params are not supported for object-specified trackers, these are already initialized!"+ )return selfdef initialize_from(- self,- persister: BasicStatePersistence,- resume_at_next_action: bool,- default_state: dict,- default_entry_point: str,+ self,+ initializer: BaseStateLoader,+ resume_at_next_action: bool,+ default_state: dict,+ default_entrypoint: str,) -> "ApplicationBuilder":- self.persister = persister+ """Initializes the application we will build from some prior state object.+ Note that you can *either* call this or use `with_state` and `with_entrypoint` -- this also assigns application ID,+ partition key, and sequence ID.++ :param initializer: The persister object to use for initialization. Likely the same one called with ``with_state_persister``.+ :param resume_at_next_action: Whether to resume at the next action, or default to the ``default_entrypoint``+ :param default_state: The default state to use if it does not exist. This is a dictionary.+ :param default_entrypoint: The default entry point to use if it does not exist or you elect not to resume_at_next_action.+ :return: The application builder for future chaining.+ """+ if self.start is not None or self.state is not None:+ raise ValueError(+ "Cannot call initialize_from if you have already set state or an entrypoint! "+ "You can either use the initializer *or* set the state and entrypoint manually."+ )+ self.initializer = initializerself.resume_at_next_action = resume_at_next_actionself.default_state = default_state- self.start = default_entry_point+ self.start = default_entrypointreturn selfdef with_state_persister(- self, persister: BasicStatePersistence, on_every: str = "step"+ self, persister: Union[BaseStateSaver, LifecycleAdapter], on_every: str = "step") -> "ApplicationBuilder":"""Adds a state persister to the application. This is a way to persist state out to a database, file, etc...- at the specified interval.+ at the specified interval. This is one of two options:++ 1. [normal mode] A BaseStateSaver object -- this is a utility class that makes it easy to save/load+ 2. [power-user-mode] A lifecycle adapter -- this is a custom class that you use to save state.++ The framework will wrap the BaseStateSaver object in a PersisterHook, which is a post-run.:param persister: The persister to add:param on_every: The interval to persist state. Currently only "step" is supported.@@ -1256,34 +1316,10 @@ class ApplicationBuilder:"""if on_every != "step":raise ValueError(f"on_every {on_every} not supported")- # stick persister into a post_step_hook if it doesn't implement it already.- if hasattr(persister, "post_step_hook"):- if persister not in self.lifecycle_adapters:- self.lifecycle_adapters.append(persister)+ if not isinstance(persister, persistence.BaseStateSaver):+ self.lifecycle_adapters.append(persister)else:- # TODO: pull this out more formally- class persisterWrapper(PostRunStepHook):- def post_run_step(- self,- app_id: str,- partition_key: str,- sequence_id: int,- state: "State",- action: "Action",- result: Optional[Dict[str, Any]],- exception: Exception,- **future_kwargs: Any,- ):- if exception is None:- persister.save(- partition_key, app_id, sequence_id, action.name, state, "completed"- )- else:- persister.save(- partition_key, app_id, sequence_id, action.name, state, "failed"- )-- self.lifecycle_adapters.append(persisterWrapper())+ self.lifecycle_adapters.append(persistence.PersisterHook(persister))return selfdef _load_from_persister(self):@@ -1296,11 +1332,11 @@ class ApplicationBuilder:"""# load state from persister- load_result = self.persister.load(self.partition_key, self.app_id, self.sequence_id)+ load_result = self.initializer.load(self.partition_key, self.app_id, self.sequence_id)if load_result is None:# there was nothing to load -- use default stateself.state = self.state.update(**self.default_state)- self.sequence_id = 0+ self.sequence_id = None # has to start at Noneelse:# there was somethinglast_position = load_result["position"]@@ -1334,7 +1370,7 @@ class ApplicationBuilder:_validate_transitions(self.transitions, all_actions)_validate_app_id(self.app_id)- if self.persister:+ if self.initializer:# sets state, sequence_id, and maybe startself._load_from_persister()_validate_start(self.start, all_actions)@@ -1351,7 +1387,7 @@ class ApplicationBuilder:],state=self.state,initial_step=self.start,- id=self.app_id,+ uid=self.app_id,partition_key=self.partition_key,sequence_id=self.sequence_id,adapter_set=LifecycleAdapterSet(*self.lifecycle_adapters),diff --git a/burr/core/persistence.py b/burr/core/persistence.pynew file mode 100644index 00000000..03bd3ec8--- /dev/null+++ b/burr/core/persistence.py@@ -0,0 +1,272 @@+import abc+import json+import sqlite3+from abc import ABCMeta+from typing import Any, Dict, Literal, Optional, TypedDict++from burr.core import Action+from burr.core.state import State, logger+from burr.lifecycle import PostRunStepHook+++class PersistedStateData(TypedDict):+ partition_key: str+ app_id: str+ sequence_id: int+ position: str+ state: State+ created_at: str+ status: str+++class BaseStateLoader(abc.ABC):+ """Base class for state initialization. This goes together with a BaseStateSaver to form the+ database for your application."""++ @abc.abstractmethod+ def load(+ self, partition_key: str, app_id: Optional[str], sequence_id: Optional[int] = None+ ) -> Optional[PersistedStateData]:+ """Loads the state for a given app_id++ :param partition_key: the partition key. Note this could be None, but it's up to the persistor to whether+ that is a valid value it can handle.+ :param app_id: the identifier for the app instance being recorded.+ :param sequence_id: optional, the state corresponding to a specific point in time.+ If not provided, should return the latest.+ :return: position, state, sequence_id+ """+ pass++ @abc.abstractmethod+ def list_app_ids(self, partition_key: str) -> list[str]:+ """Returns list of app IDs for a given primary key"""+ pass+++class BaseStateSaver(abc.ABC):+ """Basic Interface for state writing. This goes together with a BaseStateLoader to form the+ database for your application.+ """++ def initialize(self):+ """Initializes the app for saving, set up any databases, etc.. you want to here."""+ pass++ @abc.abstractmethod+ def save(+ self,+ partition_key: Optional[str],+ app_id: str,+ sequence_id: int,+ position: str,+ state: State,+ status: Literal["completed", "failed"],+ ):+ """Saves the state for a given app_id, sequence_id, position++ (PK, App_id, sequence_id, position) are a unique identifier for the state. Why not just+ (PK, App_id, sequence_id)? Because we're over-engineering this here. We're always going to have+ a position so might as well make it a quadruple.++ :param partition_key: the partition key. Note this could be None, but it's up to the persistor to whether+ that is a valid value it can handle.+ :param app_id: Appliaction UID to write with+ :param sequence_id: Sequence ID of the last executed step+ :param position: The action name that was implemented+ :param state: The current state of the application+ :param status: The status of this state, either "completed" or "failed". If "failed" the state is what it was+ before the action was applied.+ """+ pass+++class BaseStatePersister(BaseStateLoader, BaseStateSaver, metaclass=ABCMeta):+ """Utility interface for a state reader/writer. This both persists and initializes state.+ Extend this class if you want an easy way to implement custom state storage.+ """++ pass+++class PersisterHook(PostRunStepHook):+ """Wrapper class for bridging the persistence interface with lifecycle hooks. This is used internally."""++ def __init__(self, persister: BaseStateSaver):+ self.persister = persister++ def post_run_step(+ self,+ app_id: str,+ partition_key: str,+ sequence_id: int,+ state: "State",+ action: "Action",+ result: Optional[Dict[str, Any]],+ exception: Exception,+ **future_kwargs: Any,+ ):+ if exception is None:+ self.persister.save(partition_key, app_id, sequence_id, action.name, state, "completed")+ else:+ self.persister.save(partition_key, app_id, sequence_id, action.name, state, "failed")+++class SQLLitePersister(BaseStatePersister):+ """Class for SQLLite persistence of state. This is a simple implementation."""++ def __init__(self, db_path: str, table_name: str = "burr_state"):+ """Constructor++ :param db_path: the path the DB will be stored.+ :param table_name: the table name to store things under.+ """+ self.db_path = db_path+ self.table_name = table_name+ self.connection = sqlite3.connect(db_path)++ def create_table_if_not_exists(self, table_name: str):+ """Helper function to create the table where things are stored if it doesn't exist."""+ cursor = self.connection.cursor()++ cursor.execute(+ f"""+ CREATE TABLE IF NOT EXISTS {table_name} (+ partition_key TEXT NOT NULL,+ app_id TEXT NOT NULL,+ sequence_id INTEGER NOT NULL,+ position TEXT NOT NULL,+ status TEXT NOT NULL,+ state TEXT NOT NULL,+ created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,+ PRIMARY KEY (partition_key, app_id, sequence_id, position, state)+ )"""+ )+ cursor.execute(+ f"""+ CREATE INDEX IF NOT EXISTS {table_name}_created_at_index ON {table_name} (created_at);+ """+ )+ self.connection.commit()++ def initialize(self):+ """Creates the table if it doesn't exist"""+ # Usage+ self.create_table_if_not_exists(self.table_name)++ def list_app_ids(self, partition_key: str) -> list[str]:+ cursor = self.connection.cursor()+ cursor.execute(+ f"SELECT DISTINCT app_id FROM {self.table_name} "+ f"WHERE partition_key = ? "+ f"ORDER BY created_at DESC",+ (partition_key,),+ )+ app_ids = [row[0] for row in cursor.fetchall()]+ return app_ids++ def load(+ self, partition_key: str, app_id: str, sequence_id: int = None+ ) -> Optional[PersistedStateData]:+ """Loads state for a given partition id.++ Depending on the parameters, this will return the last thing written, the last thing written for a given app_id,+ or a specific sequence_id for a given app_id.++ :param partition_key:+ :param app_id:+ :param sequence_id:+ :return:+ """+ logger.debug("Loading %s, %s, %s", partition_key, app_id, sequence_id)+ cursor = self.connection.cursor()+ if app_id is None:+ # get latest for all app_ids+ cursor.execute(+ f"SELECT position, state, sequence_id, app_id, created_at, status FROM {self.table_name} "+ f"WHERE partition_key = ? "+ f"ORDER BY CREATED_AT DESC LIMIT 1",+ (partition_key,),+ )+ elif sequence_id is None:+ cursor.execute(+ f"SELECT position, state, sequence_id, app_id, created_at, status FROM {self.table_name} "+ f"WHERE partition_key = ? AND app_id = ? "+ f"ORDER BY sequence_id DESC LIMIT 1",+ (partition_key, app_id),+ )+ else:+ cursor.execute(+ f"SELECT position, state, seqeuence_id, app_id, created_at, status FROM {self.table_name} "+ f"WHERE partition_key = ? AND app_id = ? AND sequence_id = ?",+ (partition_key, app_id, sequence_id),+ )+ row = cursor.fetchone()+ if row is None:+ return None+ _state = State(json.loads(row[1])["_state"])+ return {+ "partition_key": partition_key,+ "app_id": row[3],+ "sequence_id": row[2],+ "position": row[0],+ "state": _state,+ "created_at": row[4],+ "status": row[5],+ }++ def save(+ self,+ partition_key: str,+ app_id: str,+ sequence_id: int,+ position: str,+ state: State,+ status: Literal["completed", "failed"],+ ):+ """+ Saves the state for a given app_id, sequence_id, and position.++ This method connects to the SQLite database, converts the state to a JSON string, and inserts a new record+ into the table with the provided partition_key, app_id, sequence_id, position, and state. After the operation,+ it commits the changes and closes the connection to the database.++ :param partition_key: The partition key. This could be None, but it's up to the persistor to whether+ that is a valid value it can handle.+ :param app_id: The identifier for the app instance being recorded.+ :param sequence_id: The state corresponding to a specific point in time.+ :param position: The position in the sequence of states.+ :param state: The state to be saved, an instance of the State class.+ :param status: The status of this state, either "completed" or "failed". If "failed" the state is what it was+ before the action was applied.+ :return: None+ """+ logger.debug(+ "saving %s, %s, %s, %s, %s, %s",+ partition_key,+ app_id,+ sequence_id,+ position,+ state,+ status,+ )+ cursor = self.connection.cursor()+ json_state = json.dumps(state.__dict__)+ cursor.execute(+ f"INSERT INTO {self.table_name} (partition_key, app_id, sequence_id, position, state, status) "+ f"VALUES (?, ?, ?, ?, ?, ?)",+ (partition_key, app_id, sequence_id, position, json_state, status),+ )+ self.connection.commit()++ def __del__(self):+ # closes connection at end when things are being shutdown.+ self.connection.close()+++if __name__ == "__main__":+ s = SQLLitePersister(db_path=".sqllite.db", table_name="test1")+ s.initialize()+ s.save("pk", "app_id", 1, "pos", State({"a": 1, "b": 2}), "completed")+ print(s.list_app_ids("pk"))+ print(s.load("pk", "app_id"))diff --git a/burr/core/state.py b/burr/core/state.pyindex b70b409f..ec9ecdc2 100644--- a/burr/core/state.py+++ b/burr/core/state.py@@ -1,10 +1,8 @@import abcimport copyimport dataclasses-import jsonimport logging-import sqlite3-from typing import Any, Dict, Iterator, Literal, Mapping, Optional, TypedDict+from typing import Any, Dict, Iterator, Mappinglogger = logging.getLogger(__name__)@@ -198,231 +196,3 @@ class State(Mapping):def __repr__(self):return self.get_all().__repr__() # quick hack---class PersistenceDict(TypedDict):- partition_key: str- app_id: str- sequence_id: int- position: str- state: State- created_at: str- status: str---class BasicStatePersistence(abc.ABC):- """Basic Interface for state persistence.-- A persister's job is to store and retrieve state.- """-- def initialize(self):- """Initializes the persistence."""- pass-- @abc.abstractmethod- def list_app_ids(self, partition_key: str) -> list[str]:- """Returns list of app IDs for a given primary key"""- pass-- @abc.abstractmethod- def load(- self, partition_key: str, app_id: Optional[str], sequence_id: Optional[int] = None- ) -> Optional[PersistenceDict]:- """Loads the state for a given app_id-- :param partition_key: the partition key. Note this could be None, but it's up to the persistor to whether- that is a valid value it can handle.- :param app_id: the identifier for the app instance being recorded.- :param sequence_id: optional, the state corresponding to a specific point in time.- If not provided, should return the latest.- :return: position, state, sequence_id- """- pass-- @abc.abstractmethod- def save(- self,- partition_key: Optional[str],- app_id: str,- sequence_id: int,- position: str,- state: State,- status: Literal["completed", "failed"],- ):- """Saves the state for a given app_id, sequence_id, position-- (PK, App_id, sequence_id, position) are a unique identifier for the state. Why not just- (PK, App_id, sequence_id)? Because we're over-engineering this here. We're always going to have- a position so might as well make it a quadruple.-- :param partition_key: the partition key. Note this could be None, but it's up to the persistor to whether- that is a valid value it can handle.- :param app_id:- :param sequence_id:- :param position:- :param state:- :param status:- """- pass---class SQLLitePersistence(BasicStatePersistence):- """Class for SQLLite persistence of state. This is a simple implementation."""-- def __init__(self, db_path: str, table_name: str = "burr_state"):- """Constructor-- :param db_path: the path the DB will be stored.- :param table_name: the table name to store things under.- """- self.db_path = db_path- self.table_name = table_name- self.connection = sqlite3.connect(db_path)-- @staticmethod- def create_table_if_not_exists(db_path: str, table_name: str, conn: sqlite3.Connection):- """Helper function to create the table where things are stored if it doesn't exist."""- cursor = conn.cursor()-- cursor.execute(- f"""- CREATE TABLE IF NOT EXISTS {table_name} (- partition_key TEXT NOT NULL,- app_id TEXT NOT NULL,- sequence_id INTEGER NOT NULL,- position TEXT NOT NULL,- status TEXT NOT NULL,- state TEXT NOT NULL,- created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,- PRIMARY KEY (partition_key, app_id, sequence_id, position, state)- )"""- )- cursor.execute(- f"""- CREATE INDEX IF NOT EXISTS {table_name}_created_at_index ON {table_name} (created_at);- """- )- conn.commit()-- def initialize(self):- """Creates the table if it doesn't exist"""- # Usage- self.create_table_if_not_exists(self.db_path, self.table_name, self.connection)-- def list_app_ids(self, partition_key: str) -> list[str]:- cursor = self.connection.cursor()- cursor.execute(- f"SELECT DISTINCT app_id FROM {self.table_name} "- f"WHERE partition_key = ? "- f"ORDER BY created_at DESC",- (partition_key,),- )- app_ids = [row[0] for row in cursor.fetchall()]- return app_ids-- def load(- self, partition_key: str, app_id: str, sequence_id: int = None- ) -> Optional[PersistenceDict]:- """Loads state for a given partition id.-- Depending on the parameters, this will return the last thing written, the last thing written for a given app_id,- or a specific sequence_id for a given app_id.-- :param partition_key:- :param app_id:- :param sequence_id:- :return:- """- logger.debug("Loading %s, %s, %s", partition_key, app_id, sequence_id)- cursor = self.connection.cursor()- if app_id is None:- # get latest for all app_ids- cursor.execute(- f"SELECT position, state, sequence_id, app_id, created_at, status FROM {self.table_name} "- f"WHERE partition_key = ? "- f"ORDER BY CREATED_AT DESC LIMIT 1",- (partition_key,),- )- elif sequence_id is None:- cursor.execute(- f"SELECT position, state, sequence_id, app_id, created_at, status FROM {self.table_name} "- f"WHERE partition_key = ? AND app_id = ? "- f"ORDER BY sequence_id DESC LIMIT 1",- (partition_key, app_id),- )- else:- cursor.execute(- f"SELECT position, state, seqeuence_id, app_id, created_at, status FROM {self.table_name} "- f"WHERE partition_key = ? AND app_id = ? AND sequence_id = ?",- (partition_key, app_id, sequence_id),- )- row = cursor.fetchone()- if row is None:- return None- _state = State(json.loads(row[1])["_state"])- return {- "partition_key": partition_key,- "app_id": row[3],- "sequence_id": row[2],- "position": row[0],- "state": _state,- "created_at": row[4],- "status": row[5],- }-- def save(- self,- partition_key: str,- app_id: str,- sequence_id: int,- position: str,- state: State,- status: Literal["completed", "failed"],- ):- """- Saves the state for a given app_id, sequence_id, and position.-- This method connects to the SQLite database, converts the state to a JSON string, and inserts a new record- into the table with the provided partition_key, app_id, sequence_id, position, and state. After the operation,- it commits the changes and closes the connection to the database.-- :param partition_key: The partition key. This could be None, but it's up to the persistor to whether- that is a valid value it can handle.- :param app_id: The identifier for the app instance being recorded.- :param sequence_id: The state corresponding to a specific point in time.- :param position: The position in the sequence of states.- :param state: The state to be saved, an instance of the State class.- :param status: The status of this state, either "completed" or "failed". If "failed" the state is what it was- before the action was applied.- :return: None- """- logger.debug(- "saving %s, %s, %s, %s, %s, %s",- partition_key,- app_id,- sequence_id,- position,- state,- status,- )- cursor = self.connection.cursor()- json_state = json.dumps(state.__dict__)- cursor.execute(- f"INSERT INTO {self.table_name} (partition_key, app_id, sequence_id, position, state, status) "- f"VALUES (?, ?, ?, ?, ?, ?)",- (partition_key, app_id, sequence_id, position, json_state, status),- )- self.connection.commit()-- def __del__(self):- # closes connection at end when things are being shutdown.- self.connection.close()---if __name__ == "__main__":- s = SQLLitePersistence(db_path=".sqllite.db", table_name="test1")- s.initialize()- s.save("pk", "app_id", 1, "pos", State({"a": 1, "b": 2}), "completed")- print(s.list_app_ids("pk"))- print(s.load("pk", "app_id"))diff --git a/burr/tracking/client.py b/burr/tracking/client.pyindex 8c2d0583..48875f5a 100644--- a/burr/tracking/client.py+++ b/burr/tracking/client.py@@ -3,11 +3,10 @@ import jsonimport loggingimport osimport traceback-import uuid-from typing import Any, Dict, Literal, Optional+from typing import Any, Dict, Optionalfrom burr.core import Action, ApplicationGraph, State-from burr.core.state import BasicStatePersistence, PersistenceDict+from burr.core.persistence import BaseStateLoader, PersistedStateDatafrom burr.integrations.base import require_pluginfrom burr.lifecycle import (PostApplicationCreateHook,@@ -49,7 +48,7 @@ class LocalTrackingClient(PostRunStepHook,PreStartSpanHook,PostEndSpanHook,- BasicStatePersistence,+ BaseStateLoader,):"""Tracker to track locally -- goes along with the Burr UI. Writesdown the following:@@ -65,7 +64,6 @@ class LocalTrackingClient(self,project: str,storage_dir: str = DEFAULT_STORAGE_DIR,- app_id: Optional[str] = None,):"""Instantiates a local tracking client. This will create the following directories, if they don't exist:#. The base directory (defaults to ~/.burr)@@ -80,13 +78,9 @@ class LocalTrackingClient(:param app_id: Unique application ID. If not provided, a random one will be generated. If this already exists,it will use that one/append to the files in that one."""- if app_id is None:- app_id = f"app_{str(uuid.uuid4())}"- storage_dir = LocalTrackingClient.get_storage_path(project, storage_dir)- self.app_id = app_id- self.storage_dir = storage_dir- self._ensure_dir_structure()- self.f = open(os.path.join(self.storage_dir, self.app_id, self.LOG_FILENAME), "a")++ self.f = None+ self.storage_dir = LocalTrackingClient.get_storage_path(project, storage_dir)@classmethoddef get_storage_path(cls, project, storage_dir):@@ -147,19 +141,26 @@ class LocalTrackingClient(del prior_state[key]return prior_state, entry_point- def _ensure_dir_structure(self):+ def _ensure_dir_structure(self, app_id: str):if not os.path.exists(self.storage_dir):logger.info(f"Creating storage directory: {self.storage_dir}")os.makedirs(self.storage_dir)- application_path = os.path.join(self.storage_dir, self.app_id)+ application_path = os.path.join(self.storage_dir, app_id)if not os.path.exists(application_path):logger.info(f"Creating application directory: {application_path}")os.makedirs(application_path)def post_application_create(- self, *, state: "State", application_graph: "ApplicationGraph", **future_kwargs: Any+ self,+ *,+ app_id: str,+ state: "State",+ application_graph: "ApplicationGraph",+ **future_kwargs: Any,):- path = os.path.join(self.storage_dir, self.app_id, self.GRAPH_FILENAME)+ self._ensure_dir_structure(app_id)+ self.f = open(os.path.join(self.storage_dir, app_id, self.LOG_FILENAME), "a")+ path = os.path.join(self.storage_dir, app_id, self.GRAPH_FILENAME)if os.path.exists(path):logger.info(f"Graph already exists at {path}. Not overwriting.")return@@ -242,11 +243,8 @@ class LocalTrackingClient(self._append_write_line(end_span_model)def __del__(self):- self.f.close()-- def initialize(self):- # not needed because this one does things in the constructor- pass+ if self.f is not None:+ self.f.close()def list_app_ids(self, partition_key: str) -> list[str]:# TODO:@@ -254,19 +252,7 @@ class LocalTrackingClient(def load(self, partition_key: str, app_id: Optional[str], sequence_id: Optional[int] = None- ) -> Optional[PersistenceDict]:- # TODO:- pass-- def save(- self,- partition_key: Optional[str],- app_id: str,- sequence_id: int,- position: str,- state: State,- status: Literal["completed", "failed"],- ):+ ) -> Optional[PersistedStateData]:# TODO:passdiff --git a/tests/core/test_application.py b/tests/core/test_application.pyindex abd7b4ea..c112e3dd 100644--- a/tests/core/test_application.py+++ b/tests/core/test_application.py@@ -464,10 +464,9 @@ def test_app_step():state=State({}),initial_step="counter",partition_key="test",- id="test-123",+ uid="test-123",sequence_id=0,)- assert app.sequence_id == 0action, result, state = app.step()assert app.sequence_id == 1assert action.name == "counter"@@ -484,7 +483,7 @@ def test_app_step_with_inputs():state=State({"count": 0, "tracker": []}),initial_step="counter",partition_key="test",- id="test-123",+ uid="test-123",sequence_id=0,)action, result, state = app.step(inputs={"additional_increment": 1})@@ -502,7 +501,7 @@ def test_app_step_with_inputs_missing():state=State({"count": 0, "tracker": []}),initial_step="counter",partition_key="test",- id="test-123",+ uid="test-123",sequence_id=0,)with pytest.raises(ValueError, match="missing required inputs"):@@ -518,7 +517,7 @@ def test_app_step_broken(caplog):state=State({}),initial_step="broken_action_unique_name",partition_key="test",- id="test-123",+ uid="test-123",sequence_id=0,)with caplog.at_level(logging.ERROR): # it should say the name, that's the only contract for now@@ -536,7 +535,7 @@ def test_app_step_done():state=State({}),initial_step="counter",partition_key="test",- id="test-123",+ uid="test-123",sequence_id=0,)app.step()@@ -552,10 +551,9 @@ async def test_app_astep():state=State({}),initial_step="counter_async",partition_key="test",- id="test-123",+ uid="test-123",sequence_id=0,)- assert app.sequence_id == 0action, result, state = await app.astep()assert app.sequence_id == 1assert action.name == "counter_async"@@ -572,7 +570,7 @@ async def test_app_astep_with_inputs():state=State({"count": 0, "tracker": []}),initial_step="counter_async",partition_key="test",- id="test-123",+ uid="test-123",sequence_id=0,)action, result, state = await app.astep(inputs={"additional_increment": 1})@@ -590,7 +588,7 @@ async def test_app_astep_with_inputs_missing():state=State({"count": 0, "tracker": []}),initial_step="counter_async",partition_key="test",- id="test-123",+ uid="test-123",sequence_id=0,)with pytest.raises(ValueError, match="missing required inputs"):@@ -606,7 +604,7 @@ async def test_app_astep_broken(caplog):state=State({}),initial_step="broken_action_unique_name",partition_key="test",- id="test-123",+ uid="test-123",sequence_id=0,)with caplog.at_level(logging.ERROR): # it should say the name, that's the only contract for now@@ -624,7 +622,7 @@ async def test_app_astep_done():state=State({}),initial_step="counter_async",partition_key="test",- id="test-123",+ uid="test-123",sequence_id=0,)await app.astep()@@ -640,7 +638,7 @@ def test_app_many_steps():state=State({}),initial_step="counter",partition_key="test",- id="test-123",+ uid="test-123",sequence_id=0,)action, result = None, None@@ -658,7 +656,7 @@ async def test_app_many_a_steps():state=State({}),initial_step="counter_async",partition_key="test",- id="test-123",+ uid="test-123",sequence_id=0,)action, result = None, None@@ -680,7 +678,7 @@ def test_iterate():state=State({}),initial_step="counter",partition_key="test",- id="test-123",+ uid="test-123",sequence_id=0,)res = []@@ -718,7 +716,7 @@ def test_iterate_with_inputs():state=State({}),initial_step="counter",partition_key="test",- id="test-123",+ uid="test-123",sequence_id=0,)gen = app.iterate(@@ -745,10 +743,9 @@ async def test_aiterate():state=State({}),initial_step="counter",partition_key="test",- id="test-123",+ uid="test-123",sequence_id=0,)- assert app.sequence_id == 0gen = app.aiterate(halt_after=["result"])assert app.sequence_id == 0counter = 0@@ -776,7 +773,7 @@ async def test_aiterate_halt_before():state=State({}),initial_step="counter",partition_key="test",- id="test-123",+ uid="test-123",sequence_id=0,)gen = app.aiterate(halt_before=["result"])@@ -804,7 +801,7 @@ async def test_app_aiterate_with_inputs():state=State({}),initial_step="counter",partition_key="test",- id="test-123",+ uid="test-123",sequence_id=0,)gen = app.aiterate(halt_after=["result"], inputs={"additional_increment": 10})@@ -827,7 +824,7 @@ def test_run():state=State({}),initial_step="counter",partition_key="test",- id="test-123",+ uid="test-123",sequence_id=0,)action, result, state = app.run(halt_after=["result"])@@ -847,7 +844,7 @@ def test_run_halt_before():state=State({}),initial_step="counter",partition_key="test",- id="test-123",+ uid="test-123",sequence_id=0,)action, result, state = app.run(halt_before=["result"])@@ -868,7 +865,7 @@ def test_run_with_inputs():state=State({}),initial_step="counter",partition_key="test",- id="test-123",+ uid="test-123",sequence_id=0,)action, result, state = app.run(halt_after=["result"], inputs={"additional_increment": 10})@@ -888,7 +885,7 @@ async def test_arun():state=State({}),initial_step="counter",partition_key="test",- id="test-123",+ uid="test-123",sequence_id=0,)action, result, state = await app.arun(halt_after=["result"])@@ -908,7 +905,7 @@ async def test_arun_halt_before():state=State({}),initial_step="counter",partition_key="test",- id="test-123",+ uid="test-123",sequence_id=0,)action, result, state = await app.arun(halt_before=["result"])@@ -929,7 +926,7 @@ async def test_arun_with_inputs():state=State({}),initial_step="counter",partition_key="test",- id="test-123",+ uid="test-123",sequence_id=0,)action, result, state = await app.arun(@@ -953,7 +950,7 @@ async def test_app_a_run_async_and_sync():state=State({}),initial_step="counter_sync",partition_key="test",- id="test-123",+ uid="test-123",sequence_id=0,)action, result, state = await app.arun(halt_after=["result"])@@ -974,7 +971,7 @@ def test_stream_result_halt_after():initial_step="counter",adapter_set=LifecycleAdapterSet(action_tracker),partition_key="test",- id="test-123",+ uid="test-123",sequence_id=0,)action, streaming_container = app.stream_result(halt_after=["counter_2"])@@ -1002,7 +999,7 @@ def test_stream_result_halt_after_single_step():initial_step="counter",adapter_set=LifecycleAdapterSet(action_tracker),partition_key="test",- id="test-123",+ uid="test-123",sequence_id=0,)action, streaming_container = app.stream_result(halt_after=["counter_2"])@@ -1034,7 +1031,7 @@ def test_stream_result_halt_after_run_through_final_streaming():initial_step="counter_non_streaming",adapter_set=LifecycleAdapterSet(action_tracker),partition_key="test",- id="test-123",+ uid="test-123",sequence_id=0,)action, streaming_container = app.stream_result(halt_after=["counter_streaming"])@@ -1069,7 +1066,7 @@ def test_stream_result_halt_after_run_through_final_non_streaming():initial_step="counter_non_streaming",adapter_set=LifecycleAdapterSet(action_tracker),partition_key="test",- id="test-123",+ uid="test-123",sequence_id=0,)action, streaming_container = app.stream_result(halt_after=["counter_final_non_streaming"])@@ -1102,7 +1099,7 @@ def test_stream_result_halt_before():state=State({"count": 0}),initial_step="counter_non_streaming",partition_key="test",- id="test-123",+ uid="test-123",sequence_id=0,)action, streaming_container = app.stream_result(halt_after=[], halt_before=["counter_final"])@@ -1122,7 +1119,7 @@ def test_app_set_state():state=State(),initial_step="counter",partition_key="test",- id="test-123",+ uid="test-123",sequence_id=0,)assert "counter" not in app.state # initial value@@ -1147,7 +1144,7 @@ def test_app_get_next_step():state=State(),initial_step="counter_1",partition_key="test",- id="test-123",+ uid="test-123",sequence_id=0,)# uninitialized -- counter_1@@ -1255,7 +1252,7 @@ def test_application_run_step_hooks_sync():initial_step="counter",adapter_set=internal.LifecycleAdapterSet(tracker),partition_key="test",- id="test-123",+ uid="test-123",sequence_id=0,)app.run(halt_after=["result"])@@ -1299,7 +1296,7 @@ async def test_application_run_step_hooks_async():initial_step="counter",adapter_set=internal.LifecycleAdapterSet(tracker),partition_key="test",- id="test-123",+ uid="test-123",sequence_id=0,)await app.arun(halt_after=["result"])@@ -1340,7 +1337,7 @@ async def test_application_run_step_runs_hooks():initial_step="counter",adapter_set=internal.LifecycleAdapterSet(*hooks),partition_key="test",- id="test-123",+ uid="test-123",sequence_id=0,)await app.astep()@@ -1406,7 +1403,7 @@ def test_application_post_application_create_hook():initial_step="counter",adapter_set=internal.LifecycleAdapterSet(tracker),partition_key="test",- id="test-123",+ uid="test-123",sequence_id=0,)assert "state" in tracker.called_args@@ -1426,7 +1423,7 @@ async def test_application_gives_graph():state=State({}),initial_step="counter",partition_key="test",- id="test-123",+ uid="test-123",sequence_id=0,)graph = app.graphdiff --git a/tests/core/test_persistence.py b/tests/core/test_persistence.pynew file mode 100644index 00000000..c8753d9f--- /dev/null+++ b/tests/core/test_persistence.py@@ -0,0 +1,35 @@+import pytest++from burr.core import State+from burr.core.persistence import SQLLitePersister+++def persistence():+ return SQLLitePersister(db_path=":memory:", table_name="test_table")+++def test_persistence_initialization_creates_table(persistence):+ persistence.initialize()+ assert persistence.list_app_ids("partition_key") == []+++def test_persistence_saves_and_loads_state(persistence):+ persistence.initialize()+ persistence.save("partition_key", "app_id", 1, "position", State({"key": "value"}), "status")+ loaded_state = persistence.load("partition_key", "app_id")+ assert loaded_state["state"] == State({"key": "value"})+++def test_persistence_returns_none_when_no_state(persistence):+ persistence.initialize()+ loaded_state = persistence.load("partition_key", "app_id")+ assert loaded_state is None+++def test_persistence_lists_app_ids(persistence):+ persistence.initialize()+ persistence.save("partition_key", "app_id1", 1, "position", State({"key": "value"}), "status")+ persistence.save("partition_key", "app_id2", 1, "position", State({"key": "value"}), "status")+ app_ids = persistence.list_app_ids("partition_key")+ assert set(app_ids) == set(["app_id1", "app_id2"])diff --git a/tests/core/test_state.py b/tests/core/test_state.pyindex 71046339..fa813051 100644--- a/tests/core/test_state.py+++ b/tests/core/test_state.py@@ -1,6 +1,6 @@import pytest-from burr.core.state import SQLLitePersistence, State+from burr.core.state import Statedef test_state_access():@@ -80,34 +80,3 @@ def test_state_wipe_keep():state = State({"foo": "bar", "baz": "qux"})wiped = state.wipe(keep=["foo"])assert wiped.get_all() == {"foo": "bar"}---def persistence():- return SQLLitePersistence(db_path=":memory:", table_name="test_table")---def test_persistence_initialization_creates_table(persistence):- persistence.initialize()- assert persistence.list_app_ids("partition_key") == []---def test_persistence_saves_and_loads_state(persistence):- persistence.initialize()- persistence.save("partition_key", "app_id", 1, "position", State({"key": "value"}), "status")- loaded_state = persistence.load("partition_key", "app_id")- assert loaded_state["state"] == State({"key": "value"})---def test_persistence_returns_none_when_no_state(persistence):- persistence.initialize()- loaded_state = persistence.load("partition_key", "app_id")- assert loaded_state is None---def test_persistence_lists_app_ids(persistence):- persistence.initialize()- persistence.save("partition_key", "app_id1", 1, "position", State({"key": "value"}), "status")- persistence.save("partition_key", "app_id2", 1, "position", State({"key": "value"}), "status")- app_ids = persistence.list_app_ids("partition_key")- assert set(app_ids) == set(["app_id1", "app_id2"])diff --git a/tests/tracking/test_local_tracking_client.py b/tests/tracking/test_local_tracking_client.pyindex ca5cce43..f897cafa 100644--- a/tests/tracking/test_local_tracking_client.py+++ b/tests/tracking/test_local_tracking_client.py@@ -37,9 +37,8 @@ def sample_application(project_name: str, log_dir: str, app_id: str, broken: boo("counter", "result", default),).with_entrypoint("counter")- .with_tracker(- project=project_name, tracker="local", params={"storage_dir": log_dir, "app_id": app_id}- )+ .with_tracker(project=project_name, tracker="local", params={"storage_dir": log_dir})+ .with_identifiers(app_id=app_id).build())