Build applications that make decisions (chatbots, agents, simulations, etc...). Monitor, trace, persist, and execute on your own infrastructure.
Python aiburrchatbot-frameworkdagsgenerative-aigraphshacktoberfestllmopsllmsmlopspersistent-data-structurestate-machinestate-managementvisibility
Wires through persistence to spawned children apps
elijahbenizzy25c907bcd2d9
diff --git a/burr/common/async_utils.py b/burr/common/async_utils.pyindex 94e573cd..edda13a7 100644--- a/burr/common/async_utils.py+++ b/burr/common/async_utils.py@@ -4,7 +4,6 @@ from typing import AsyncGenerator, AsyncIterable, Generator, List, TypeVar, UnioT = TypeVar("T")GenType = TypeVar("GenType")-ReturnType = TypeVar("ReturnType")SyncOrAsyncIterable = Union[AsyncIterable[T], List[T]]SyncOrAsyncGenerator = Union[Generator[GenType, None, None], AsyncGenerator[GenType, None]]@@ -31,7 +30,7 @@ async def arealize(maybe_async_generator: SyncOrAsyncGenerator[GenType]) -> List"""Realize an async generator or async iterable to a list.:param maybe_async_generator: async generator or async iterable- :return: list+ :return: list of items -- fully realized"""if inspect.isasyncgen(maybe_async_generator):out = [item async for item in maybe_async_generator]diff --git a/burr/common/types.py b/burr/common/types.pyindex 3a26d00e..6b6b2d18 100644--- a/burr/common/types.py+++ b/burr/common/types.py@@ -1,8 +1,14 @@+import abcimport dataclasses-from typing import Optional+from typing import Any, Optional+try:+ from typing import Self+except ImportError:+ Self = Any-# This contains commmon types++# This contains common types# Currently the types are a little closer to the logic than we'd like# We'll want to break them out into interfaces and put more here eventually# This will help avoid the ugly if TYPE_CHECKING imports;@@ -11,3 +17,11 @@ class ParentPointer:app_id: strpartition_key: Optional[str]sequence_id: Optional[int]+++class BaseCopyable(abc.ABC):+ """Interface for copying objects. This is used internally."""++ @abc.abstractmethod+ def copy(self) -> "Self":+ passdiff --git a/burr/core/application.py b/burr/core/application.pyindex 33aeb288..f312af72 100644--- a/burr/core/application.py+++ b/burr/core/application.py@@ -487,7 +487,14 @@ class ApplicationGraph(Graph):@dataclasses.dataclass-class ApplicationContext(AbstractContextManager):+class ApplicationIdentifiers:+ app_id: str+ partition_key: Optional[str]+ sequence_id: Optional[int]+++class ApplicationContext(AbstractContextManager, ApplicationIdentifiers):"""Application context. This is anything your node might need to know about the application.Often used for recursive tracking.@@ -508,11 +515,10 @@ class ApplicationContext(AbstractContextManager):"""- app_id: str- partition_key: Optional[str]- sequence_id: Optional[int]tracker: Optional["TrackingClient"]parallel_executor_factory: Callable[[], Executor]+ state_initializer: Optional[BaseStateLoader]+ state_persister: Optional[BaseStateSaver]@staticmethoddef get() -> Optional["ApplicationContext"]:@@ -718,6 +724,8 @@ class Application(Generic[ApplicationStateType]):spawning_parent_pointer: Optional[burr_types.ParentPointer] = None,tracker: Optional["TrackingClient"] = None,parallel_executor_factory: Optional[Executor] = None,+ state_persister: Union[BaseStateSaver, LifecycleAdapter, None] = None,+ state_initializer: Union[BaseStateLoader, LifecycleAdapter, None] = None,):"""Instantiates an Application. This is an internal API -- use the builder!@@ -767,6 +775,8 @@ class Application(Generic[ApplicationStateType]):"__context": self._context_factory,}self._spawning_parent_pointer = spawning_parent_pointer+ self._state_initializer = state_initializer+ self._state_persister = state_persisterself._adapter_set.call_all_lifecycle_hooks_sync("post_application_create",state=self._state,@@ -807,6 +817,8 @@ class Application(Generic[ApplicationStateType]):partition_key=self._partition_key,sequence_id=sequence_id,parallel_executor_factory=self._parallel_executor_factory,+ state_initializer=self._state_initializer,+ state_persister=self._state_persister,)def _step(@@ -1937,7 +1949,7 @@ class ApplicationBuilder(Generic[StateType]):self.app_id: str = str(uuid.uuid4())self.partition_key: Optional[str] = Noneself.sequence_id: Optional[int] = None- self.initializer = None+ self.state_initializer = Noneself.use_entrypoint_from_save_state: Optional[bool] = Noneself.default_state: Optional[dict] = Noneself.fork_from_app_id: Optional[str] = None@@ -1951,7 +1963,8 @@ class ApplicationBuilder(Generic[StateType]):self.graph_builder = Noneself.prebuilt_graph = Noneself.typing_system = None- self._parallel_executor_factory = None+ self.parallel_executor_factory = None+ self.state_persister = Nonedef with_identifiers(self, app_id: str = None, partition_key: str = None, sequence_id: int = None@@ -1996,7 +2009,7 @@ class ApplicationBuilder(Generic[StateType])::param kwargs: Key-value pairs to set in the state:return: The application builder for future chaining."""- if self.initializer is not None:+ if self.state_initializer is not None:raise ValueError(BASE_ERROR_MESSAGE + "You cannot set state if you are loading state""the .initialize_from() API. Either allow the persister to set the "@@ -2062,14 +2075,14 @@ class ApplicationBuilder(Generic[StateType])::param executor::return:"""- if self._parallel_executor_factory is not None:+ if self.parallel_executor_factory is not None:raise ValueError(BASE_ERROR_MESSAGE+ "You have already set an executor. You cannot set multiple executors. Current executor is:"- f"{self._parallel_executor_factory}"+ f"{self.parallel_executor_factory}")- self._parallel_executor_factory = executor_factory+ self.parallel_executor_factory = executor_factoryreturn selfdef _ensure_no_prebuilt_graph(self):@@ -2238,7 +2251,7 @@ class ApplicationBuilder(Generic[StateType]):+ "If you set fork_from_partition_key or fork_from_sequence_id, you must also set fork_from_app_id. ""See .initialize_from() documentation.")- self.initializer = initializer+ self.state_initializer = initializerself.resume_at_next_action = resume_at_next_actionself.default_state = default_stateself.start = default_entrypoint@@ -2278,6 +2291,7 @@ class ApplicationBuilder(Generic[StateType]):except NotImplementedError:passself.lifecycle_adapters.append(persistence.PersisterHook(persister))+ self.state_persister = persister # track for laterreturn selfdef with_spawning_parent(@@ -2326,11 +2340,11 @@ class ApplicationBuilder(Generic[StateType]):_app_id = self.app_id_sequence_id = self.sequence_id# load state from persister- load_result = self.initializer.load(_partition_key, _app_id, _sequence_id)+ load_result = self.state_initializer.load(_partition_key, _app_id, _sequence_id)if load_result is None:if self.fork_from_app_id is not None:logger.warning(- f"{self.initializer.__class__.__name__} returned None while trying to fork from: "+ f"{self.state_initializer.__class__.__name__} returned None while trying to fork from: "f"partition_key:{_partition_key}, app_id:{_app_id}, "f"sequence_id:{_sequence_id}. ""You explicitly requested to fork from a prior application run, but it does not exist. "@@ -2344,7 +2358,7 @@ class ApplicationBuilder(Generic[StateType]):if load_result["state"] is None:raise ValueError(BASE_ERROR_MESSAGE- + f"Error: {self.initializer.__class__.__name__} returned {load_result} for "+ + f"Error: {self.state_initializer.__class__.__name__} returned {load_result} for "f"partition_key:{_partition_key}, app_id:{_app_id}, "f"sequence_id:{_sequence_id}, ""but value for state was None! This is not allowed. Please return just None in this case, "@@ -2395,7 +2409,7 @@ class ApplicationBuilder(Generic[StateType]):_validate_app_id(self.app_id)if self.state is None:self.state = State()- if self.initializer:+ if self.state_initializer:# sets state, sequence_id, and maybe startself._load_from_persister()graph = self._get_built_graph()@@ -2432,5 +2446,7 @@ class ApplicationBuilder(Generic[StateType]):if self.spawn_from_app_id is not Noneelse None),- parallel_executor_factory=self._parallel_executor_factory,+ parallel_executor_factory=self.parallel_executor_factory,+ state_persister=self.state_persister,+ state_initializer=self.state_initializer,)diff --git a/burr/core/parallelism.py b/burr/core/parallelism.pyindex 329c93b9..dd431c38 100644--- a/burr/core/parallelism.py+++ b/burr/core/parallelism.py@@ -1,12 +1,31 @@import abcimport asyncioimport dataclasses+import hashlibimport inspect-from typing import Any, AsyncGenerator, Callable, Dict, Generator, List, Tuple, TypeVar, Union-+from typing import (+ Any,+ AsyncGenerator,+ Callable,+ Dict,+ Generator,+ List,+ Literal,+ Optional,+ Tuple,+ TypeVar,+ Union,+)++from burr.common import async_utils+from burr.common.async_utils import SyncOrAsyncGenerator, SyncOrAsyncGeneratorOrItemOrListfrom burr.core import Action, Application, ApplicationBuilder, ApplicationContext, Graph, Statefrom burr.core.action import SingleStepAction+from burr.core.application import ApplicationIdentifiersfrom burr.core.graph import GraphBuilder+from burr.core.persistence import BaseStateLoader, BaseStateSaver+from burr.lifecycle import LifecycleAdapter+from burr.tracking.base import TrackingClientSubgraphType = Union[Action, Callable, "RunnableGraph"]@@ -42,6 +61,13 @@ class RunnableGraph:return RunnableGraph(graph=graph, entrypoint=action.name, halt_after=[action.name])+TrackerBehavior = Union[Literal["cascade"], None, TrackingClient]+StatePersisterBehavior = Union[Literal["cascade"], BaseStateSaver, LifecycleAdapter, None]+StateInitializerBehavior = Union[Literal["cascade"], BaseStateLoader, LifecycleAdapter, None]++AdapterType = TypeVar("AdapterType", bound=Union[BaseStateSaver, BaseStateLoader, LifecycleAdapter])++@dataclasses.dataclassclass SubGraphTask:"""Task to run a subgraph. Has runtime-spefici information, like inputs, state, and@@ -53,26 +79,47 @@ class SubGraphTask:inputs: Dict[str, Any]state: Stateapplication_id: str+ # The following require you to specify a tracking client, base state saver, etc...+ tracker: Optional[TrackingClient] = None+ state_persister: Optional[BaseStateSaver] = None+ state_initializer: Optional[BaseStateLoader] = None- def _create_app(self, parent_context: ApplicationContext) -> Application:- return (+ def _create_app(self, parent_context: ApplicationIdentifiers) -> Application:+ builder = (ApplicationBuilder().with_graph(self.graph.graph)- .with_entrypoint(self.graph.entrypoint)- .with_state(self.state).with_spawning_parent(app_id=parent_context.app_id,sequence_id=parent_context.sequence_id,partition_key=parent_context.partition_key,)- .with_tracker(parent_context.tracker.copy()) # We have to copy# TODO -- handle persistence....with_identifiers(app_id=self.application_id,partition_key=parent_context.partition_key, # cascade the partition key)- .build())+ if self.tracker is not None:+ builder = builder.with_tracker(self.tracker) # TODO -- move this into the adapter+ # In this case we want to persist the state for the app+ if self.state_persister is not None:+ builder = builder.with_state_persister(self.state_persister)+ # In this case we want to initialize from it+ # We're going to use default settings (initialize from the latest+ # TODO -- consider if there's a case in which we want to initialize+ # in a custom manner+ # if state_initializer is not None and self.cascade_state_initializer:+ if self.state_initializer is not None:+ builder = builder.initialize_from(+ self.state_initializer,+ default_state=self.state.get_all(), # TODO _- ensure that any hidden variables aren't used...+ default_entrypoint=self.graph.entrypoint,+ resume_at_next_action=True,+ )+ else:+ builder = builder.with_entrypoint(self.graph.entrypoint).with_state(self.state)++ return builder.build()def run(self,@@ -99,11 +146,11 @@ def _stable_app_id_hash(app_id: str, child_key: str) -> str:"""Gives a stable hash for an application. Given the parent app_id and a child key,this will give a hash that will be stable across runs.- :param app_id:- :param additional_key:- :return:+ :param app_id: Parent application ID+ :param child_key: Child key to hash+ :return: Stable hash of the parent app_id and child key"""- ...+ return hashlib.sha256(f"{app_id}:{child_key}".encode()).hexdigest()class TaskBasedParallelAction(SingleStepAction):@@ -163,6 +210,12 @@ class TaskBasedParallelAction(SingleStepAction):})return state.update(all_llm_outputs=all_llm_outputs)++ Note that it can be synchronous *or* asynchronous. Synchronous implementations will use the standard/+ supplied executor. Asynchronous implementations will use asyncio.gather. Note that, while asynchronous+ implementations may implement the tasks as either synchronous or asynchronous generators, synchronous implementations+ can only use synchronous generators. Furthermore, with asynchronous implementations, the generator for reduce+ will be asynchronous as well (regardless of whether your task functions are asynchronous)."""def __init__(self):@@ -208,15 +261,12 @@ class TaskBasedParallelAction(SingleStepAction):)task_generator = self.tasks(state_without_internals, context, run_kwargs)- # TODO -- run in parallelasync def state_generator():"""This makes it easier on the user -- if they don't have an async generator we can still exhause itThis way we run through all of the task generators. These correspond to the task generation capabilities above (the map*/task generation stuff)"""- if inspect.isasyncgen(task_generator):- coroutines = [task.arun(context) async for task in task_generator]- else:- coroutines = [task.arun(context) for task in task_generator]+ all_tasks = await async_utils.arealize(task_generator)+ coroutines = [item.arun(context) for item in all_tasks]results = await asyncio.gather(*coroutines)# TODO -- yield in order...for result in results:@@ -248,7 +298,7 @@ class TaskBasedParallelAction(SingleStepAction):@abc.abstractmethoddef tasks(self, state: State, context: ApplicationContext, inputs: Dict[str, Any]- ) -> Generator[SubGraphTask, None, None]:+ ) -> SyncOrAsyncGenerator[SubGraphTask]:"""Creates all tasks that this action will run, given the state/inputs.This produces a generator of SubGraphTasks that will be run in parallel.@@ -259,7 +309,7 @@ class TaskBasedParallelAction(SingleStepAction):pass@abc.abstractmethod- def reduce(self, state: State, states: Generator[State, None, None]) -> State:+ def reduce(self, state: State, states: SyncOrAsyncGenerator[State]) -> State:"""Reduces the states from the tasks into a single state.:param states: State outputs from the subtasks@@ -278,6 +328,31 @@ class TaskBasedParallelAction(SingleStepAction):pass+def _cascade_adapter(+ behavior: Union[Literal["cascade"], AdapterType, None],+ adapter: Union[AdapterType, None],+) -> Union[AdapterType, None]:+ """Cascades the desired adapter given the specified behavior (cascade, None, or an adapter).+ If shared_instance is specified (non-null) as well as `cascade`, then the shared instance will be used,+ *if* it is the same instance as the specified adapter.++ This allows us to ensure that sharing, say, persisters will result in the same++ :param behavior:+ :param adapter:+ :param copy_from:+ :return:+ """+ if behavior is None:+ return None+ # TODO -- consider checking this in a cleaner way+ elif behavior == "cascade":+ if hasattr(adapter, "copy"):+ adapter = adapter.copy()+ return adapter+ return behavior++class MapActionsAndStates(TaskBasedParallelAction):"""Base class to run a cartesian product of actions x states.@@ -348,7 +423,7 @@ class MapActionsAndStates(TaskBasedParallelAction):@abc.abstractmethoddef actions(self, state: State, context: ApplicationContext, inputs: Dict[str, Any]- ) -> Generator[SubgraphType, None, None]:+ ) -> SyncOrAsyncGenerator[SubgraphType]:"""Yields actions to run in parallel. These will be merged with the states as a cartesian product.:param state: Input state at the time of running the "parent" action.@@ -360,7 +435,7 @@ class MapActionsAndStates(TaskBasedParallelAction):@abc.abstractmethoddef states(self, state: State, context: ApplicationContext, inputs: Dict[str, Any]- ) -> Generator[State, None, None]:+ ) -> SyncOrAsyncGenerator[State]:"""Yields states to run in parallel. These will be merged with the actions as a cartesian product.:param state: Input state at the time of running the "parent" action.@@ -372,7 +447,7 @@ class MapActionsAndStates(TaskBasedParallelAction):def tasks(self, state: State, context: ApplicationContext, inputs: Dict[str, Any]- ) -> Generator[SubGraphTask, None, None]:+ ) -> SyncOrAsyncGenerator[SubGraphTask]:"""Takes the cartesian product of actions and states, creating tasks for each.:param state: Input state at the time of running the "parent" action.@@ -380,16 +455,53 @@ class MapActionsAndStates(TaskBasedParallelAction)::param inputs: Runtime Inputs to the action:return: Generator of tasks to run"""- for i, action in enumerate(self.actions(state, context, inputs)):- for j, state in enumerate(self.states(state, context, inputs)):- key = f"{i}-{j}" # this is a stable hash for now but will not handle caching- # TODO -- allow for custom hashes that will indicate stability (user is responsible)- yield SubGraphTask(- graph=RunnableGraph.create(action),- inputs=inputs,- state=state,- application_id=_stable_app_id_hash(context.app_id, key),- )++ def _create_task(key: str, action: Action, substate: State) -> SubGraphTask:+ tracker = _cascade_adapter(self.tracker(), context.tracker)+ state_initializer_behavior = self.state_initializer()+ state_initializer = _cascade_adapter(+ self.state_initializer(), context.state_initializer+ )+ state_persister_behavior = self.state_persister()+ # In this case we want to ensure they share the same instance as they do in the parent+ # This enables us to mirror expected behavior, and is the standard case+ # Specifically, that the persister is the same as the initializer+ if (+ state_initializer_behavior == "cascade"+ and state_persister_behavior == "cascade"+ and context.state_persister is context.state_initializer+ ):+ state_persister = state_initializer+ # In the case that they are not the same, we want to cascade the persister separately+ else:+ state_persister = _cascade_adapter(self.state_persister(), context.state_persister)+ return SubGraphTask(+ graph=RunnableGraph.create(action),+ inputs=inputs,+ state=substate,+ application_id=_stable_app_id_hash(context.app_id, key),+ tracker=tracker,+ state_persister=state_persister,+ state_initializer=state_initializer,+ )++ def _tasks() -> Generator[SubGraphTask, None, None]:+ for i, action in enumerate(self.actions(state, context, inputs)):+ for j, substate in enumerate(self.states(state, context, inputs)):+ key = f"{i}-{j}" # this is a stable hash for now but will not handle caching+ yield _create_task(key, action, substate)++ async def _atasks() -> AsyncGenerator[SubGraphTask, None]:+ action_generator = async_utils.asyncify_generator(self.actions(state, context, inputs))+ state_generator = async_utils.asyncify_generator(self.states(state, context, inputs))+ actions = await async_utils.arealize(action_generator)+ states = await async_utils.arealize(state_generator)+ for i, action in enumerate(actions):+ for j, substate in enumerate(states):+ key = f"{i}-{j}"+ yield _create_task(key, action, substate)++ return _atasks() if self.is_async() else _tasks()@abc.abstractmethoddef reduce(self, state: State, states: Generator[State, None, None]) -> State:@@ -400,6 +512,53 @@ class MapActionsAndStates(TaskBasedParallelAction):"""pass+ def state_persister(self, **kwargs) -> StatePersisterBehavior:+ """Persister for the action -- what persister does the sub-application use?++ This can either be:+ - "cascade": inherit from parent+ - None: don't use a persister+ - Object: use the specified persister++ Note this is global -- if you want to override it on a per-subgraph basis, you'll need to use the task-level API.++ :param kwargs: Additional arguments, reserverd for later+ :return: The specified behavior.+ """+ return "cascade"++ def state_initializer(self, **kwargs) -> StateInitializerBehavior:+ """State initializer for the action -- what initializer does the sub-application use?++ This can either be:+ - "cascade": inherit from parent+ - None: don't use an initializer+ - Object: use the specified initializer++ Note this is global -- if you want to override it on a per-subgraph basis, you'll need to use the task-level API.++ :param kwargs: Additional arguments, reserverd for later++ :return: the specified behavior+ """+ return "cascade"++ def tracker(self, **kwargs) -> TrackerBehavior:+ """Tracker for the action -- what tracker does the sub-application use?++ This can either be:+ - "cascade": inherit from parent+ - None: don't use a tracker+ - Object: use the specified tracker++ Note this is global -- if you want to override it on a per-subgraph basis, you'll need to use the task-level API.+++ :param kwargs: Additional arguments, reserverd for later+ :return: the specified behavior+ """+ return "cascade"+class MapActions(MapActionsAndStates, abc.ABC):"""Base class to run a set of actions over the same state. Actions can be functions (decorated with @action),@@ -460,7 +619,7 @@ class MapActions(MapActionsAndStates, abc.ABC):@abc.abstractmethoddef actions(self, state: State, inputs: Dict[str, Any], context: ApplicationContext- ) -> Generator[SubgraphType, None, None]:+ ) -> SyncOrAsyncGenerator[SubgraphType]:"""Gives all actions to map over, given the state/inputs.:param state: State at the time of running the action@@ -486,7 +645,7 @@ class MapActions(MapActionsAndStates, abc.ABC):yield self.state(state, inputs)@abc.abstractmethod- def reduce(self, state: State, states: Generator[State, None, None]) -> State:+ def reduce(self, state: State, states: SyncOrAsyncGenerator[State]) -> State:"""Reduces the task's results into a single state. Runs through all outputsand combines them together, to form the final state for the action.@@ -558,7 +717,7 @@ class MapStates(MapActionsAndStates, abc.ABC):@abc.abstractmethoddef states(self, state: State, context: ApplicationContext, inputs: Dict[str, Any]- ) -> Generator[State, None, None]:+ ) -> SyncOrAsyncGenerator[State]:"""Generates all states to map over, given the state and inputs.Each state will be an update to the input state.@@ -593,13 +752,13 @@ class MapStates(MapActionsAndStates, abc.ABC):def actions(self, state: State, context: ApplicationContext, inputs: Dict[str, Any]- ) -> Generator[SubgraphType, None, None]:+ ) -> SyncOrAsyncGenerator[SubgraphType]:"""Maps the action over each state generated by the `states` method.Internally used, do not implement."""yield self.action(state, inputs)@abc.abstractmethod- def reduce(self, state: State, results: Generator[State, None, None]) -> State:+ def reduce(self, state: State, results: SyncOrAsyncGenerator[State]) -> State:"""Reduces the task's results:param results:@@ -608,13 +767,6 @@ class MapStates(MapActionsAndStates, abc.ABC):pass-GenType = TypeVar("GenType")-ReturnType = TypeVar("ReturnType")--SyncOrAsyncGenerator = Union[Generator[GenType, None, None], AsyncGenerator[GenType, None]]-SyncOrAsyncGeneratorOrItemOrList = Union[SyncOrAsyncGenerator[GenType], List[GenType], GenType]--class PassThroughMapActionsAndStates(MapActionsAndStates):def __init__(self,@@ -641,7 +793,7 @@ class PassThroughMapActionsAndStates(MapActionsAndStates):def actions(self, state: State, context: ApplicationContext, inputs: Dict[str, Any]- ) -> Generator[SubgraphType, None, None]:+ ) -> SyncOrAsyncGenerator[SubgraphType]:if isinstance(self._action_or_generator, list):for action in self._action_or_generator:yield actiondiff --git a/burr/core/persistence.py b/burr/core/persistence.pyindex 71eeff51..f2af19f2 100644--- a/burr/core/persistence.py+++ b/burr/core/persistence.py@@ -6,10 +6,16 @@ from abc import ABCMetafrom collections import defaultdictfrom typing import Any, Dict, Literal, Optional, TypedDict+from burr.common.types import BaseCopyablefrom burr.core import Actionfrom burr.core.state import State, loggerfrom burr.lifecycle import PostRunStepHook+try:+ from typing import Self+except ImportError:+ Self = None+class PersistedStateData(TypedDict):partition_key: str@@ -142,8 +148,16 @@ class DevNullPersister(BaseStatePersister):return-class SQLLitePersister(BaseStatePersister):- """Class for SQLLite persistence of state. This is a simple implementation."""+class SQLitePersister(BaseStatePersister, BaseCopyable):+ """Class for SQLite persistence of state. This is a simple implementation."""++ def copy(self) -> "Self":+ return SQLitePersister(+ db_path=self.db_path,+ table_name=self.table_name,+ serde_kwargs=self.serde_kwargs,+ connect_kwargs=self._connect_kwargs,+ )PARTITION_KEY_DEFAULT = ""@@ -169,6 +183,7 @@ class SQLLitePersister(BaseStatePersister):)self.serde_kwargs = serde_kwargs or {}self._initialized = False+ self._connect_kwargs = connect_kwargsdef create_table_if_not_exists(self, table_name: str):"""Helper function to create the table where things are stored if it doesn't exist."""@@ -176,7 +191,7 @@ class SQLLitePersister(BaseStatePersister):cursor.execute(f"""CREATE TABLE IF NOT EXISTS {table_name} (- partition_key TEXT DEFAULT '{SQLLitePersister.PARTITION_KEY_DEFAULT}',+ partition_key TEXT DEFAULT '{SQLitePersister.PARTITION_KEY_DEFAULT}',app_id TEXT NOT NULL,sequence_id INTEGER NOT NULL,position TEXT NOT NULL,@@ -215,7 +230,7 @@ class SQLLitePersister(BaseStatePersister):def list_app_ids(self, partition_key: Optional[str], **kwargs) -> list[str]:partition_key = (- partition_key if partition_key is not None else SQLLitePersister.PARTITION_KEY_DEFAULT+ partition_key if partition_key is not None else SQLitePersister.PARTITION_KEY_DEFAULT)cursor = self.connection.cursor()@@ -246,7 +261,7 @@ class SQLLitePersister(BaseStatePersister)::return:"""partition_key = (- partition_key if partition_key is not None else SQLLitePersister.PARTITION_KEY_DEFAULT+ partition_key if partition_key is not None else SQLitePersister.PARTITION_KEY_DEFAULT)logger.debug("Loading %s, %s, %s", partition_key, app_id, sequence_id)cursor = self.connection.cursor()@@ -322,7 +337,7 @@ class SQLLitePersister(BaseStatePersister):status,)partition_key = (- partition_key if partition_key is not None else SQLLitePersister.PARTITION_KEY_DEFAULT+ partition_key if partition_key is not None else SQLitePersister.PARTITION_KEY_DEFAULT)cursor = self.connection.cursor()json_state = json.dumps(state.serialize(**self.serde_kwargs))@@ -391,8 +406,10 @@ class InMemoryPersister(BaseStatePersister):self._storage[partition_key][app_id].append(persisted_state)+SQLLitePersister = SQLitePersister+if __name__ == "__main__":- s = SQLLitePersister(db_path=".sqllite.db", table_name="test1")+ s = SQLitePersister(db_path=".SQLite.db", table_name="test1")s.initialize()s.save("pk", "app_id", 1, "pos", State({"a": 1, "b": 2}), "completed")print(s.list_app_ids("pk"))diff --git a/burr/tracking/client.py b/burr/tracking/client.pyindex f9070e18..aab7c9c0 100644--- a/burr/tracking/client.py+++ b/burr/tracking/client.py@@ -2,6 +2,7 @@ import abcimport dataclassesimport datetime+from burr.common.types import BaseCopyablefrom burr.lifecycle.base import (DoLogAttributeHook,PostEndStreamHook,@@ -119,6 +120,7 @@ class SyncTrackingClient(PreStartStreamHook,PostStreamItemHook,PostEndStreamHook,+ BaseCopyable,ABC,):@abc.abstractmethoddiff --git a/docs/concepts/parallelism.rst b/docs/concepts/parallelism.rstindex 32815245..81e8013c 100644--- a/docs/concepts/parallelism.rst+++ b/docs/concepts/parallelism.rst@@ -485,14 +485,39 @@ Persistence/TrackingBy default, the trackers/persisters will be passed from the parent application to the child application. The application IDswill be created as a a stable hash of the parent ID + the index of the child ID, requiring the order to be constant to ensure that the same application ID is used for the same task every time.-It will also utilize the same persister to load from the prior state, if that is used on the application level (see the state-persistence section).+Note that we will be adding hte ability to create a unique key, per application, but that is not implemented yet.++It will also utilize the same persister to load from the prior state, if that is used on the application level (see :ref:`state persistence <state-persistence>`).This enables the following:1. Tracking will automatically be associated with the same application (and sub-application) when reloaded-2. If the concurrent application quits halfway through, bthe application will be able to pick up where it left off, as will all sub-applications+2. If the concurrent application quits halfway through, the application will be able to pick up where it left off, as will all sub-applications++The default behavior is to cascade all the following constructs through to the parallel child applications:++1. Tracking (given by the :py:meth:`burr.core.application.ApplicationBuilder.with_tracker` function in :py:class:`ApplicationBuilder <burr.core.application.ApplicationBuilder>`)+2. State saving/persistence (given by the :py:meth:`burr.core.application.ApplicationBuilder.with_state_persister` function in :py:class:`burr.core.application.ApplicationBuilder`)+3. State loading (given by the :py:meth:`burr.core.application.ApplicationBuilder.initialize_from` function in :py:class:`burr.core.application.ApplicationBuilder`)++These will then be passed ot the corresponding sub-application tasks. If any of these objects implement a `.copy()` method, that will be called. Otherwise the same instance will be created. Not the tracker/persister/loader must be serializable+to work in non-multithreaded executors, and thread-safe to work in multithreaded instances.++In mst cases, (2) and (3) above will be the same, however there all valid use-cases where you might want to use different persisters/loaders for the sub-application.+Thus, the default will be to pass the same instance for (2) and (3), *if* they both exist and are specified.-You can disable either tracking or persistence at the sub-application level by passing ``track=False`` or ``persist=False`` to the constructor of the parallel action superclass.+You can disable either tracking or persistence at the sub-application level by overriding one of the three methods (corresponding to the three adapter-types above),+on any of the :py:class:`burr.core.parallelism.MapStatesAndActions`, :py:class:`burr.core.parallelism.MapStates`, or :py:class:`burr.core.parallelism.MapActions` classes.++If you return `None`, then no tracker/persister will be used. If you return an object (E.G. a tracker or a persister), that will be used instead.+The function will be called for every instance of the sub-application.++1. Overriding the tracker :py:meth:`burr.core.parallelism.MapStatesAndAction.tracker`+2. Overriding the persister :py:meth:`burr.core.parallelism.MapStatesAndAction.state_persister`+3. Overriding the loader :py:meth:`burr.core.parallelism.MapStatesAndAction.state_initializer`++You can also set the corresponding fields in the :py:class:`burr.core.parallelism.SubGraphTask` object on a per-task basis,+if you are using the lower-level API.You can also disable it globally using the application builder:@@ -503,18 +528,20 @@ You can also disable it globally using the application builder:def action(self) -> Action | Callable | RunnableGraph:return runnable_graph- def tracker(self, context: ApplicationContext) -> TrackingBehavior | None:+ def tracker(self, context) -> TrackingBehavior | None:# return "cascade" # default# return None # no trackingreturn LocalTrackingClient(...) # custom tracker- def persister(self, context: ApplicationContext) -> Persister | None:+ def state_persister(self) -> Persister | None:# return "cascade" # default# return None # no persistencereturn SQLLitePersister(...) # custom persister- ... # same as above+ def state_initializer(self) -> StateInitializer | None:+ return self.state_perssiter() # same as the persister+ ... # same as aboveOther-----diff --git a/tests/core/test_parallelism.py b/tests/core/test_parallelism.pyindex b3c84fb0..6404414b 100644--- a/tests/core/test_parallelism.py+++ b/tests/core/test_parallelism.py@@ -1,8 +1,9 @@import asyncio+import concurrent.futuresimport dataclassesimport datetimefrom random import random-from typing import Any, AsyncGenerator, Callable, Dict, Generator, List, Optional, Union+from typing import Any, AsyncGenerator, Callable, Dict, Generator, List, Literal, Optional, Unionimport pytest@@ -24,8 +25,10 @@ from burr.core.parallelism import (RunnableGraph,SubGraphTask,TaskBasedParallelAction,+ _cascade_adapter,map_reduce_action,)+from burr.core.persistence import BaseStateLoader, BaseStateSaver, PersistedStateDatafrom burr.tracking.base import SyncTrackingClientfrom burr.visibility import ActionSpan@@ -33,7 +36,7 @@ old_action = actionasync def sleep_random():- await asyncio.sleep(random())+ await asyncio.sleep(random() / 1000)# Single action/callable subgraph@@ -212,8 +215,13 @@ class RecursiveActionTracked:class RecursiveActionTracker(SyncTrackingClient):"""Simple test tracking client for a recursive action"""- def __init__(self, events: List[RecursiveActionTracked]):+ def __init__(+ self,+ events: List[RecursiveActionTracked],+ parent: Optional["RecursiveActionTracker"] = None,+ ):self.events = events+ self.parent = parentdef copy(self):"""Quick way to copy from the current state. This assumes linearity (which is true in this case, as parallelism is delegated)"""@@ -221,7 +229,7 @@ class RecursiveActionTracker(SyncTrackingClient):current_event = self.events[-1]if current_event.state_after is not None:raise ValueError("Don't copy if you're not in the middle of an event")- return RecursiveActionTracker(current_event.children)+ return RecursiveActionTracker(current_event.children, parent=self)raise ValueError("Don't copy if you're not in the middle of an event")def post_application_create(@@ -609,7 +617,7 @@ def test_e2e_map_actions_and_states_sync():"""Tests the map states action with a subgraph that is run in parallel.Collatz conjecture over different starting points"""- class MapStatesAsync(MapActionsAndStates):+ class MapStatesSync(MapActionsAndStates):def actions(self, state: State, context: ApplicationContext, inputs: Dict[str, Any]) -> Generator[Union[Action, Callable, RunnableGraph], None, None]:@@ -649,7 +657,7 @@ def test_e2e_map_actions_and_states_sync():ApplicationBuilder().with_actions(initial_action=Input("input_numbers_in_state"),- map_action=MapStatesAsync(),+ map_action=MapStatesSync(),final_action=Result("output_numbers_in_state"),).with_transitions(("initial_action", "map_action"), ("map_action", "final_action"))@@ -770,6 +778,7 @@ def test_task_level_API_e2e_sync():inputs={},state=state.update(input_number=input_number, number_to_add=10),application_id=f"{i}_{j}",+ tracker=context.tracker.copy(),)def reduce(self, state: State, states: Generator[State, None, None]) -> State:@@ -841,6 +850,7 @@ async def test_task_level_API_e2e_async():inputs={},state=state.update(input_number=input_number, number_to_add=10),application_id=f"{i}_{j}",+ tracker=context.tracker.copy(),)async def reduce(self, state: State, states: AsyncGenerator[State, None]) -> State:@@ -943,3 +953,237 @@ def test_map_reduce_function_e2e():_, map_event, __ = eventsgrouped_events = _group_events_by_app_id(map_event.children)assert len(grouped_events) == 9 # cartesian product of 3 actions and 3 states+++class DummyTracker(SyncTrackingClient):+ def __init__(self, parent: Optional["DummyTracker"] = None):+ self.parent = parent++ def copy(self):+ return DummyTracker(parent=self)++ def post_application_create(+ self,+ *,+ app_id: str,+ partition_key: Optional[str],+ state: "State",+ application_graph: "ApplicationGraph",+ parent_pointer: Optional[burr_types.ParentPointer],+ spawning_parent_pointer: Optional[burr_types.ParentPointer],+ **future_kwargs: Any,+ ):+ pass++ def pre_run_step(+ self,+ *,+ app_id: str,+ partition_key: str,+ sequence_id: int,+ state: "State",+ action: "Action",+ inputs: Dict[str, Any],+ **future_kwargs: Any,+ ):+ pass++ def post_run_step(+ self,+ *,+ app_id: str,+ partition_key: str,+ sequence_id: int,+ state: "State",+ action: "Action",+ result: Optional[Dict[str, Any]],+ exception: Exception,+ **future_kwargs: Any,+ ):+ pass++ def pre_start_span(+ self,+ *,+ action: str,+ action_sequence_id: int,+ span: "ActionSpan",+ span_dependencies: list[str],+ app_id: str,+ partition_key: Optional[str],+ **future_kwargs: Any,+ ):+ pass++ def post_end_span(+ self,+ *,+ action: str,+ action_sequence_id: int,+ span: "ActionSpan",+ span_dependencies: list[str],+ app_id: str,+ partition_key: Optional[str],+ **future_kwargs: Any,+ ):+ pass++ def do_log_attributes(+ self,+ *,+ attributes: Dict[str, Any],+ action: str,+ action_sequence_id: int,+ span: Optional["ActionSpan"],+ tags: dict,+ app_id: str,+ partition_key: Optional[str],+ **future_kwargs: Any,+ ):+ pass++ def pre_start_stream(+ self,+ *,+ action: str,+ sequence_id: int,+ app_id: str,+ partition_key: Optional[str],+ **future_kwargs: Any,+ ):+ pass++ def post_stream_item(+ self,+ *,+ item: Any,+ item_index: int,+ stream_initialize_time: datetime.datetime,+ first_stream_item_start_time: datetime.datetime,+ action: str,+ sequence_id: int,+ app_id: str,+ partition_key: Optional[str],+ **future_kwargs: Any,+ ):+ pass++ def post_end_stream(+ self,+ *,+ action: str,+ sequence_id: int,+ app_id: str,+ partition_key: Optional[str],+ **future_kwargs: Any,+ ):+ pass+++class DummyPersister(BaseStateSaver, BaseStateLoader):+ def __init__(self, parent: Optional["DummyPersister"] = None):+ self.parent = parent++ def copy(self) -> "DummyPersister":+ return DummyPersister(parent=self)++ def save(+ self,+ partition_key: Optional[str],+ app_id: str,+ sequence_id: int,+ position: str,+ state: State,+ status: Literal["completed", "failed"],+ **kwargs,+ ):+ pass++ def load(+ self, partition_key: str, app_id: Optional[str], sequence_id: Optional[int] = None, **kwargs+ ) -> Optional[PersistedStateData]:+ pass++ def list_app_ids(self, partition_key: str, **kwargs) -> list[str]:+ pass+++def test_cascade_adapter_cascade():+ # Tests that cascading the adapter results in a cloned adapter with `copy()` called+ adapter = DummyTracker()+ cascaded = _cascade_adapter("cascade", adapter)+ assert cascaded.parent is adapter+++def test_cascade_adapter_none():+ # Tests that setting the adapter behavior to None results in no adapter+ adapter = DummyTracker()+ cascaded = _cascade_adapter(None, adapter)+ assert cascaded is None+++def test_cascade_adapter_fixed():+ # Tests that setting the adapter behavior to a fixed value results in that value+ current_adapter = DummyTracker()+ next_adapter = DummyTracker()+ cascaded = _cascade_adapter(next_adapter, current_adapter)+ assert cascaded is next_adapter+++def test_map_actions_and_states_uses_same_persister_as_loader():+ """This tests the MapActionsAndStates functionality of using the correct persister. Specifically+ we want it to use the same instance for the saver as it does the loader, as that is+ what the parent app does."""++ class SimpleMapStates(MapActionsAndStates):+ def actions(+ self, state: State, context: ApplicationContext, inputs: Dict[str, Any]+ ) -> Generator[Union[Action, Callable, RunnableGraph], None, None]:+ for graph_ in [+ simple_single_fn_subgraph.bind(identifying_number=1000),+ ]:+ yield graph_++ def states(+ self, state: State, context: ApplicationContext, inputs: Dict[str, Any]+ ) -> Generator[State, None, None]:+ yield state.update(input_number=0, number_to_add=0)++ def reduce(self, state: State, states: Generator[State, None, None]) -> State:+ # TODO -- ensure that states is in the correct order...+ # Or decide to key it?+ new_state = state+ for output_state in states:+ new_state = new_state.append(output_numbers_in_state=output_state["output_number"])+ return new_state++ @property+ def writes(self) -> list[str]:+ return ["output_numbers_in_state"]++ @property+ def reads(self) -> list[str]:+ return ["input_numbers_in_state"]++ action = SimpleMapStates()+ persister = DummyPersister()+ tracker = DummyTracker()++ task_generator = action.tasks(+ state=State(),+ context=ApplicationContext(+ app_id="app_id",+ partition_key="partition_key",+ sequence_id=0,+ tracker=tracker,+ state_persister=persister,+ state_initializer=persister,+ parallel_executor_factory=lambda: concurrent.futures.ThreadPoolExecutor(),+ ),+ inputs={},+ )+ (task,) = task_generator # one task+ assert task.state_persister is not None+ assert task.state_initializer is not None+ assert task.tracker is not None+ assert task.state_persister is task.state_initializer # This ensures they're the samediff --git a/tests/test_end_to_end.py b/tests/test_end_to_end.pyindex 4d15730f..b309a241 100644--- a/tests/test_end_to_end.py+++ b/tests/test_end_to_end.py@@ -2,12 +2,24 @@but they're specifically meant to be a smoke-screen. If you eversee failures in these tests, you should make a unit test, demonstrate the failure there,then fix both in that test and the end-to-end test."""+import asyncio+import uuid+from concurrent.futures import ThreadPoolExecutorfrom io import StringIO-from typing import Any, Tuple+from typing import Any, AsyncGenerator, Dict, Generator, Tuplefrom unittest.mock import patch-from burr.core import Action, ApplicationBuilder, State, action+from burr.core import (+ Action,+ ApplicationBuilder,+ ApplicationContext,+ GraphBuilder,+ State,+ action,+ persistence,+)from burr.core.action import Input, Result, expr+from burr.core.parallelism import MapStates, RunnableGraph, SubgraphTypefrom burr.lifecycle import base@@ -60,6 +72,274 @@ def test_end_to_end_collatz_with_function_api():assert hook.count == 112+def test_end_to_end_parallel_collatz_many_unreliable_tasks(tmpdir):+ """Tests collatz conjecture search on multiple tasks.+ Each of these persists its own capabilities and state.+ This uses the in memory persister to store the state of each task.+ Each task is unreliable -- it will break every few times. We will restart+ to ensure it is eventually successful using a while loop.++ This simulates running a complex workflow in parallel.+ """++ MIN_NUMBER = 80+ MAX_NUMBER = 100+ FAILURE_ODDS = 0.05 # 1 in twenty chance of faiulre, will be hit but not all the time++ seen = set()++ class UnreliableFailureError(Exception):+ pass++ def _fail_at_random():+ import random++ if random.random() < FAILURE_ODDS:+ raise UnreliableFailureError("Random failure")++ # dummy as we want an initial action to decide between odd/even next+ @action(reads=["n"], writes=["n", "original_n"])+ def initial(state: State, __context: ApplicationContext) -> State:+ # This assert ensures we only visit once per app, globally+ # Thus if we're restarting this will break+ assert __context.app_id not in seen, f"App id {__context.app_id} already seen"+ seen.add(__context.app_id)+ return state.update(original_n=state["n"])++ @action(reads=["n"], writes=["n", "n_history"])+ def even(state: State) -> State:+ _fail_at_random()+ result = {"n": state["n"] // 2}+ return state.update(**result).append(n_history=result["n"])++ @action(reads=["n"], writes=["n", "n_history"])+ def odd(state: State) -> Tuple[dict, State]:+ _fail_at_random()+ result = {"n": 3 * state["n"] + 1}+ return result, state.update(**result).append(n_history=result["n"])++ collatz_graph = (+ GraphBuilder()+ .with_actions(+ initial,+ even,+ odd,+ result=Result("n_history"),+ )+ .with_transitions(+ (["initial", "even"], "result", expr("n == 1")),+ (["initial", "even", "odd"], "even", expr("n % 2 == 0")),+ (["initial", "even", "odd"], "odd", expr("n % 2 != 0")),+ )+ .build()+ )++ @action(reads=[], writes=["ns"])+ def map_step(state: State, min_number: int = MIN_NUMBER, max_number: int = MAX_NUMBER) -> State:+ return state.update(ns=list(range(min_number, max_number)))++ class ParallelCollatz(MapStates):+ def states(+ self, state: State, context: ApplicationContext, inputs: Dict[str, Any]+ ) -> Generator[State, None, None]:+ for item in state["ns"]:+ yield state.update(n=item)++ def action(self, state: State, inputs: Dict[str, Any]) -> SubgraphType:+ return RunnableGraph(+ collatz_graph,+ entrypoint="initial",+ halt_after=["result"],+ )++ def reduce(self, state: State, results: Generator[State, None, None]) -> State:+ new_state = state+ count_mapping = {}+ for result in results:+ count_mapping[result["original_n"]] = len(result["n_history"])+ return new_state.update(counts=count_mapping)++ @property+ def writes(self) -> list[str]:+ return ["counts"]++ @property+ def reads(self) -> list[str]:+ return ["ns"]++ persister = persistence.InMemoryPersister()++ app_id = f"collatz_test_{str(uuid.uuid4())}"+ final_state = None+ while final_state is None:+ try:+ containing_application = (+ ApplicationBuilder()+ .with_actions(+ map_step,+ parallel_collatz=ParallelCollatz(),+ final=Result("counts"),+ )+ .with_transitions(+ ("map_step", "parallel_collatz"),+ ("parallel_collatz", "final"),+ )+ .with_state_persister(persister)+ .with_identifiers(app_id=app_id)+ .initialize_from(+ persister,+ resume_at_next_action=True,+ default_state={},+ default_entrypoint="map_step",+ )+ # Uncomment for debugging/visualizing+ # .with_tracker("local", project="test_persister")+ .with_parallel_executor(lambda: ThreadPoolExecutor(max_workers=10))+ .build()+ )+ *_, final_state = containing_application.run(halt_after=["final"])+ except UnreliableFailureError:+ continue++ # We want to ensure that initial is called once per app+ assert len(seen) == len(range(MIN_NUMBER, MAX_NUMBER)), "Should have seen all numbers"+++async def test_end_to_end_parallel_collatz_many_unreliable_tasks_async(tmpdir):+ """Tests collatz conjecture search on multiple tasks.+ Each of these persists its own capabilities and state.+ This uses the in memory persister to store the state of each task.+ Each task is unreliable -- it will break every few times. We will restart+ to ensure it is eventually successful using a while loop.++ This simulates running a complex workflow in parallel.+ """++ MIN_NUMBER = 80+ MAX_NUMBER = 100+ FAILURE_ODDS = 0.05 # 1 in twenty chance of faiulre, will be hit but not all the time++ seen = set()++ class UnreliableFailureError(Exception):+ pass++ def _fail_at_random():+ import random++ if random.random() < FAILURE_ODDS:+ raise UnreliableFailureError("Random failure")++ # dummy as we want an initial action to decide between odd/even next+ @action(reads=["n"], writes=["n", "original_n"])+ async def initial(state: State, __context: ApplicationContext) -> State:+ # This assert ensures we only visit once per app, globally+ # Thus if we're restarting this will break+ await asyncio.sleep(0.001)+ assert __context.app_id not in seen, f"App id {__context.app_id} already seen"+ seen.add(__context.app_id)+ return state.update(original_n=state["n"])++ @action(reads=["n"], writes=["n", "n_history"])+ async def even(state: State) -> State:+ await asyncio.sleep(0.001)+ _fail_at_random()+ result = {"n": state["n"] // 2}+ return state.update(**result).append(n_history=result["n"])++ @action(reads=["n"], writes=["n", "n_history"])+ async def odd(state: State) -> Tuple[dict, State]:+ await asyncio.sleep(0.001)+ _fail_at_random()+ result = {"n": 3 * state["n"] + 1}+ return result, state.update(**result).append(n_history=result["n"])++ collatz_graph = (+ GraphBuilder()+ .with_actions(+ initial,+ even,+ odd,+ result=Result("n_history"),+ )+ .with_transitions(+ (["initial", "even"], "result", expr("n == 1")),+ (["initial", "even", "odd"], "even", expr("n % 2 == 0")),+ (["initial", "even", "odd"], "odd", expr("n % 2 != 0")),+ )+ .build()+ )++ @action(reads=[], writes=["ns"])+ def map_step(state: State, min_number: int = MIN_NUMBER, max_number: int = MAX_NUMBER) -> State:+ return state.update(ns=list(range(min_number, max_number)))++ class ParallelCollatz(MapStates):+ async def states(+ self, state: State, context: ApplicationContext, inputs: Dict[str, Any]+ ) -> AsyncGenerator[State, None]:+ for item in state["ns"]:+ yield state.update(n=item)++ def action(self, state: State, inputs: Dict[str, Any]) -> SubgraphType:+ return RunnableGraph(+ collatz_graph,+ entrypoint="initial",+ halt_after=["result"],+ )++ async def reduce(self, state: State, results: AsyncGenerator[State, None]) -> State:+ new_state = state+ count_mapping = {}+ async for result in results:+ count_mapping[result["original_n"]] = len(result["n_history"])+ return new_state.update(counts=count_mapping)++ @property+ def writes(self) -> list[str]:+ return ["counts"]++ @property+ def reads(self) -> list[str]:+ return ["ns"]++ def is_async(self) -> bool:+ return True++ persister = persistence.InMemoryPersister()+ app_id = f"collatz_test_{str(uuid.uuid4())}"+ final_state = None+ while final_state is None:+ try:+ containing_application = (+ ApplicationBuilder()+ .with_actions(+ map_step,+ parallel_collatz=ParallelCollatz(),+ final=Result("counts"),+ )+ .with_transitions(+ ("map_step", "parallel_collatz"),+ ("parallel_collatz", "final"),+ )+ .with_state_persister(persister)+ .with_identifiers(app_id=app_id)+ .initialize_from(+ persister,+ resume_at_next_action=True,+ default_state={},+ default_entrypoint="map_step",+ )+ .build()+ )+ *_, final_state = await containing_application.arun(halt_after=["final"])+ except UnreliableFailureError:+ continue++ # We want to ensure that initial is called once per app+ assert len(seen) == len(range(MIN_NUMBER, MAX_NUMBER)), "Should have seen all numbers"++def test_echo_bot():@action(reads=["prompt"], writes=["response"])def echo(state: State) -> Tuple[dict, State]: