Skip to content

armgit / burr

Public Imported from GitHub
0

Build applications that make decisions (chatbots, agents, simulations, etc...). Monitor, trace, persist, and execute on your own infrastructure.

Updates the API for streaming sync mode to match streaming async

elijahbenizzy64960f3fc10b
diff --git a/burr/core/action.py b/burr/core/action.py
index af585497..01faf56d 100644
--- a/burr/core/action.py
+++ b/burr/core/action.py
@@ -1,6 +1,5 @@
import abc
import ast
-import collections
import copy
import inspect
import sys
@@ -8,7 +7,9 @@ import types
import typing
from typing import (
Any,
+ AsyncGenerator,
Callable,
+ Coroutine,
Dict,
Generator,
Iterator,
@@ -481,12 +482,24 @@ class FunctionBasedAction(SingleStepAction):
return inspect.iscoroutinefunction(self._fn)
+StreamType = Tuple[dict, Optional[State]]
+
+GeneratorReturnType = Generator[StreamType, None, None]
+AsyncGeneratorReturnType = AsyncGenerator[StreamType, None]
+
+StreamingFn = Callable[..., GeneratorReturnType]
+StreamingFnAsync = Callable[..., AsyncGeneratorReturnType]
+
+
class StreamingAction(Action, abc.ABC):
+ """Base class for Streaming action. These are "multi-step", meaning that
+ they run in multiple passes (run -> update)"""
+
@abc.abstractmethod
- def stream_run(self, state: State, **run_kwargs) -> Generator[dict, None, Tuple[dict, State]]:
+ def stream_run(self, state: State, **run_kwargs) -> Generator[dict, None, None]:
"""Streaming action ``stream_run`` is different than standard action run. It:
- 1. streams in a result (the dict output)
- 2. Returns the final result
+ 1. streams in an intermediate result (the dict output)
+ 2. yields the final result at the end
Note that the user, in this case, is responsible for joining the result.
@@ -499,7 +512,7 @@ class StreamingAction(Action, abc.ABC):
for token in query(state['prompt']):
yield {'response' : token}
buffer.append(token)
- return {'response' : "".join(buffer)}
+ yield {'response' : "".join(buffer)}
This would utilize a simple string buffer (implemented by a list) to store the results
and then join them at the end. We return the final result.
@@ -511,12 +524,12 @@ class StreamingAction(Action, abc.ABC):
pass
def run(self, state: State, **run_kwargs) -> dict:
+ """Runs the streaming action through to completion."""
gen = self.stream_run(state, **run_kwargs)
- while True:
- try:
- next(gen) # if we just run through, we do nothing with the result
- except StopIteration as e:
- return e.value
+ last_result = None
+ for item in gen:
+ last_result = item
+ return last_result
@property
def streaming(self) -> bool:
@@ -524,11 +537,47 @@ class StreamingAction(Action, abc.ABC):
class AsyncStreamingAction(Action, abc.ABC):
+ """Asynchronous version of the streaming action. This is a base class for streaming actions.
+ Currently this is separate from the synchronous version, but we may want to merge them in the future.
+ Note this is the "multi-step" variant, in which run/update are separate."""
+
@abc.abstractmethod
- async def stream_run(self, state: State, **run_kwargs) -> typing.AsyncGenerator[dict, None]:
+ async def stream_run(self, state: State, **run_kwargs) -> AsyncGenerator[dict, None]:
+ """Asynchronous streaming action ``stream_run`` is different than the standard action run. It:
+ 1. streams in an intermediate result (the dict output)
+ 2. yields the final result at the end
+
+ Note that the user, in this case, is responsible for joining the result.
+
+ For instance, you could have:
+
+ .. code-block:: python
+
+ async def stream_run(state: State) -> Generator[dict, None, dict]:
+ buffer = [] # you might want to be more efficient than simple strcat
+ async for token in query(state['prompt']): # asynchronous generator
+ yield {'response' : token}
+ buffer.append(token)
+ yield {'response' : "".join(buffer)}
+
+ This would utilize a simple string buffer (implemented by a list) to store the results
+ and then join them at the end. We return the final result.
+
+ :param state: State to run the action on
+ :param run_kwargs: parameters passed to the run function -- these are specified by `inputs`
+ :return: A generator that streams in a result and returns the final result
+ """
pass
async def run(self, state: State, **run_kwargs) -> dict:
+ """Runs the streaming action through to completion.
+ Returns the final result. This is used if we want a streaming action
+ as an intermediate.
+
+ :param state: State to run the action on
+ :param run_kwargs: Additional arguments to the function passed at runtime.
+ :return: Final result
+ """
gen = self.stream_run(state, **run_kwargs)
result = None
async for item in gen:
@@ -558,14 +607,13 @@ class StreamingResultContainer(Iterator[dict]):
.. code-block:: python
- streaming_result_container = application.stream_result(...)
- action_we_just_ran = streaming_result_container.get()
+ action_we_just_ran, streaming_result_container = application.stream_result(...)
print(f"getting streaming results for action={action_we_just_ran.name}")
for result_component in streaming_result_container:
print(result_component['response']) # this assumes you have a response key in your result
- final_state, final_result = streaming_result_container.get()
+ final_result, final_state = streaming_result_container.get()
"""
@staticmethod
@@ -574,9 +622,8 @@ class StreamingResultContainer(Iterator[dict]):
This is to be used internally -- it allows us to wrap non-streaming action results in a streaming
result container."""
- def empty_generator() -> Generator[dict, None, tuple[dict, State]]:
- yield from ()
- return results, final_state
+ def empty_generator() -> GeneratorReturnType:
+ yield results, final_state
return StreamingResultContainer(
empty_generator(),
@@ -585,21 +632,20 @@ class StreamingResultContainer(Iterator[dict]):
lambda result, state, exc: None,
)
- def __next__(self):
- return next(self.generator())
-
def __init__(
self,
- streaming_result_generator: Generator[dict, None, tuple[dict, State]],
+ streaming_result_generator: GeneratorReturnType,
initial_state: State,
process_result: Callable[[dict, State], tuple[dict, State]],
callback: Callable[[Optional[dict], State, Optional[Exception]], None],
):
- """Initializes a ``StreamingResultContainer``. This is meant to be used internally
+ """Initializes a streaming result container. User will never call directly.
- :param streaming_result_generator: The generator that produces the streaming result
- :param initial_state: The initial state
- :param process_result: Function to process the result -- this gets called after the generator is exhausted, prior to returning the final result
+ :param streaming_result_generator: Generator of streaming results. Note that this
+ will always yield result, Optional[State] -- regardless of the API used to create it.
+ :param initial_state: The initial state
+ :param process_result: Function to process the result -- this gets called after the generator is exhausted,
+ prior to returning the final result
:param callback: Callback to call at the very end. This will only get called *once*, and will be called during the finally block of the generator
"""
self.streaming_result_generator = streaming_result_generator
@@ -610,52 +656,85 @@ class StreamingResultContainer(Iterator[dict]):
self._result = None
self._callback_realized = False
- def __iter__(self):
- return self.generator()
+ def __next__(self):
+ if self._result is not None:
+ # we're done, and we've run through it
+ raise StopIteration
+ result, state = self.streaming_result_generator.__next__()
+ if state is not None: # we're done -- we've hit the last one
+ self._result = self._process_result(result, state)
+ raise StopIteration
+ return result
- def generator(self):
- try:
- if self._result is not None:
- # this means we're already thorugh it, we can just call it a day
+ def __iter__(self):
+ def gen_fn():
+ try:
+ while True:
+ out = self.__next__()
+ yield out
+ except StopIteration:
return
- result = yield from self.streaming_result_generator
- self._result = self._process_result(*result)
- finally:
- if self._result is None:
- # This way we always have something after we exhaust the generator
- self._result = None, self._initial_state
- if not self._callback_realized:
- exc = sys.exc_info()[1]
- # For now this will not be the right exception type (Generator close),
- # but its OK -- the exception is outside of our control fllow
- self._callback_realized = True
- self._callback(*self._result, exc)
-
- def get(self) -> tuple[Optional[dict], State]:
- """Blocking call to get the final result of the streaming action. This will
- run through the entire generator (or until an exception is raised) and return
- the final result.
-
- :return: A tuple of the result and the new state
- """
- collections.deque(
- self, maxlen=0
- ) # exhausts the generator, producing the `_result` variable
+ finally:
+ if self._result is None:
+ self._result = None, self._initial_state
+ if not self._callback_realized:
+ exc = sys.exc_info()[1]
+ self._callback_realized = True
+ self._callback(*self._result, exc)
+
+ # We really don't need an internal generator function but this was done to keep it the same
+ # as the async version
+ return gen_fn()
+
+ def get(self) -> StreamType:
+ # exhaust the generator
+ for _ in self:
+ pass
+
return self._result
class AsyncStreamingResultContainer(typing.AsyncIterator[dict]):
+ """Container for an async streaming result. This allows you to:
+ 1. Iterate over the result as it comes in
+ 2. Await the final result/state at the end
+
+ If you're familiar with generators/iterators in python, this is effectively an
+ iterator that caches the final result after calling it. This is meant to be used
+ exclusively with the streaming action calls in `Application`. Note that you will
+ never instantiate this class directly, but you will use it in the API when it is returned
+ by :py:meth:`astream_result <burr.core.application.Application.stream_result>`.
+ For reference, here's how you would use it:
+
+ .. code-block:: python
+
+ action_we_just_ran, streaming_result_container = await application.stream_result(...)
+ print(f"getting streaming results for action={action_we_just_ran.name}")
+
+ async for result_component in streaming_result_container:
+ print(result_component['response']) # this assumes you have a response key in your result
+
+ final_result, final_state = await streaming_result_container.get()
+ """
+
def __init__(
self,
- streaming_result_generator: typing.AsyncGenerator[
- typing.Tuple[dict, Optional[State]], None
- ],
+ streaming_result_generator: AsyncGeneratorReturnType,
initial_state: State,
process_result: Callable[[dict, State], tuple[dict, State]],
callback: Callable[
[Optional[dict], State, Optional[Exception]], typing.Coroutine[None, None, None]
],
):
+ """Initializes an async streaming result container. User will never call directly.
+
+ :param streaming_result_generator: Generator of streaming results. Note that this
+ will always yield result, Optional[State] -- regardless of the API used to create it.
+ :param initial_state: The initial state
+ :param process_result: Function to process the result -- this gets called after the generator is exhausted,
+ prior to returning the final result
+ :param callback: Callback to call at the very end. This will only get called *once*, and will be called during the finally block of the generator
+ """
self.streaming_result_generator = streaming_result_generator
self._initial_state = initial_state
self._process_result = process_result
@@ -663,15 +742,6 @@ class AsyncStreamingResultContainer(typing.AsyncIterator[dict]):
self._result = None
self._callback_realized = False
- #
- # async def _cb(self):
- # print("calling back")
- # result = self._result if self._result is not None else None, self._initial_state
- # if not self._callback_realized:
- # exc = sys.exc_info()[1]
- # self._callback_realized = True
- # await self._callback(*result, exc)
-
async def __anext__(self):
# TODO -- ensure this works
if self._result is not None:
@@ -684,7 +754,7 @@ class AsyncStreamingResultContainer(typing.AsyncIterator[dict]):
return result
def __aiter__(self):
- async def g():
+ async def gen_fn():
try:
while True:
yield await self.__anext__()
@@ -698,9 +768,11 @@ class AsyncStreamingResultContainer(typing.AsyncIterator[dict]):
self._callback_realized = True
await self._callback(*self._result, exc)
- return g()
+ # return it as `__aiter__` cannot be async/have awaits :/
+ return gen_fn()
async def get(self) -> tuple[Optional[dict], State]:
+ # exhaust the generator
async for _ in self:
pass
@@ -708,7 +780,7 @@ class AsyncStreamingResultContainer(typing.AsyncIterator[dict]):
@staticmethod
def pass_through(results: dict, final_state: State) -> "AsyncStreamingResultContainer":
- async def just_results() -> typing.AsyncGenerator[tuple[dict, State], None]:
+ async def just_results() -> AsyncGeneratorReturnType:
yield results, final_state
async def empty_callback(result: Optional[dict], state: State, exc: Optional[Exception]):
@@ -726,75 +798,59 @@ class SingleStepStreamingAction(SingleStepAction, abc.ABC):
"""
@abc.abstractmethod
- async def stream_run_and_update(
+ def stream_run_and_update(
self, state: State, **run_kwargs
- ) -> Generator[dict, None, tuple[dict, State]]:
+ ) -> Union[GeneratorReturnType, AsyncGeneratorReturnType]:
"""Streaming version of the run and update function. This
return type is a generator that streams in a result, has no "send"
value, and returns the final result (new result + state).
"""
pass
- def run_and_update(self, state: State, **run_kwargs) -> tuple[dict, State]:
- """Runs the action and returns the final result. This allows us to run this as a
- single step action. This is helpful for when the streaming result needs to be
- run as an intermediate.
- """
+ def _run_and_update(self, state: State, **run_kwargs) -> tuple[dict, State]:
gen = self.stream_run_and_update(state, **run_kwargs)
- while True:
- try:
- next(gen) # if we just run through, we do nothing with the result
- except StopIteration as e:
- return e.value
-
- @property
- def streaming(self) -> bool:
- return True
-
+ result = None
+ new_state = state
+ for result, new_state in gen:
+ pass
+ # TODO -- validate that it has a single length output
+ return result, new_state
-class SingleStepStreamingActionAsync(SingleStepAction, abc.ABC):
- """Class to represent a "single-step" streaming action. This is meant to
- work with the functional API. Note this is not user-facing -- the user will
- only interact with this by using the ``@streaming_action`` decorator.
- """
+ async def _arun_and_update(self, state: State, **run_kwargs) -> tuple[dict, State]:
+ gen = self.stream_run_and_update(state, **run_kwargs)
+ last_result = None
+ new_state = state
+ async for last_result, new_state in gen:
+ pass
+ return last_result, new_state
- @abc.abstractmethod
- async def stream_run_and_update(
+ def run_and_update(
self, state: State, **run_kwargs
- ) -> typing.AsyncGenerator[tuple[dict, Optional[State]], None]:
- """Streaming version of the run and update function. This
- return type is a generator that streams in a result, has no "send"
- value.
- """
- pass
-
- async def run_and_update(self, state: State, **run_kwargs) -> tuple[dict, State]:
+ ) -> Union[tuple[dict, State], Coroutine[Any, Any, tuple[dict, State]]]:
"""Runs the action and returns the final result. This allows us to run this as a
single step action. This is helpful for when the streaming result needs to be
- run as an intermediate.
- """
- gen = self.stream_run_and_update(state, **run_kwargs)
- last_result = None
- while True:
- try:
- last_result = await gen.__anext__()
- except StopAsyncIteration:
- return last_result
+ run as an intermediate."""
+ if self.is_async():
+ return self._arun_and_update(state, **run_kwargs)
+ return self._run_and_update(state, **run_kwargs)
@property
def streaming(self) -> bool:
return True
def is_async(self) -> bool:
- return True
+ return inspect.isasyncgenfunction(self.stream_run_and_update)
class FunctionBasedStreamingAction(SingleStepStreamingAction):
- _fn: Callable[..., Generator[dict, None, tuple[dict, State]]]
+ _fn: Union[StreamingFn, StreamingFnAsync]
def __init__(
self,
- fn: Callable[..., Generator[dict, None, tuple[dict, State]]],
+ fn: Union[
+ StreamingFn,
+ StreamingFnAsync,
+ ],
reads: List[str],
writes: List[str],
bound_params: dict = None,
@@ -812,10 +868,21 @@ class FunctionBasedStreamingAction(SingleStepStreamingAction):
self._writes = writes
self._bound_params = bound_params if bound_params is not None else {}
+ async def _a_stream_run_and_update(
+ self, state: State, **run_kwargs
+ ) -> AsyncGeneratorReturnType:
+ async for result in self._fn(state, **self._bound_params, **run_kwargs):
+ yield result
+
+ def _stream_run_and_update(self, state: State, **run_kwargs) -> GeneratorReturnType:
+ yield from self._fn(state, **self._bound_params, **run_kwargs)
+
def stream_run_and_update(
self, state: State, **run_kwargs
- ) -> Generator[dict, None, tuple[dict, State]]:
- return (yield from self._fn(state, **self._bound_params, **run_kwargs))
+ ) -> Union[AsyncGeneratorReturnType, GeneratorReturnType]:
+ if self.is_async():
+ return self._a_stream_run_and_update(state, **run_kwargs)
+ return self._stream_run_and_update(state, **run_kwargs)
@property
def reads(self) -> list[str]:
@@ -845,70 +912,11 @@ class FunctionBasedStreamingAction(SingleStepStreamingAction):
return _get_inputs(self._bound_params, self._fn)
@property
- def fn(self) -> Callable:
+ def fn(self) -> Union[StreamingFn, StreamingFnAsync]:
return self._fn
-
-# TODO -- merge this with the above
-class FunctionBasedStreamingActionAsync(SingleStepStreamingActionAsync):
- _fn: Callable[..., typing.AsyncGenerator[typing.Tuple[dict, Optional[State]], None]]
-
- def __init__(
- self,
- fn: Callable[..., typing.AsyncGenerator[typing.Tuple[dict, Optional[State]], None]],
- reads: List[str],
- writes: List[str],
- bound_params: dict = None,
- ):
- """Instantiates a function-based streaming action with the given function, reads, and writes.
- The function must take in a state (and inputs) and return a generator of (result, new_state).
-
- :param fn: Function to use
- :param reads:
- :param writes:
- """
- super(FunctionBasedStreamingActionAsync, self).__init__()
- self._fn = fn
- self._reads = reads
- self._writes = writes
- self._bound_params = bound_params if bound_params is not None else {}
-
- async def stream_run_and_update(
- self, state: State, **run_kwargs
- ) -> typing.AsyncGenerator[typing.Tuple[dict, Optional[State]], None]:
- async for result in self._fn(state, **self._bound_params, **run_kwargs):
- yield result
-
- @property
- def reads(self) -> list[str]:
- return self._reads
-
- @property
- def writes(self) -> list[str]:
- return self._writes
-
- @property
- def streaming(self) -> bool:
- return True
-
- def with_params(self, **kwargs: Any) -> "FunctionBasedStreamingActionAsync":
- """Binds parameters to the function. This is not user-facing -- this is
- meant to be used internally by the API.
-
- :param kwargs:
- :return:
- """
- return FunctionBasedStreamingActionAsync(
- self._fn, self._reads, self._writes, {**self._bound_params, **kwargs}
- )
-
- @property
- def inputs(self) -> tuple[list[str], list[str]]:
- return _get_inputs(self._bound_params, self._fn)
-
- @property
- def fn(self) -> Callable:
- return self._fn
+ def is_async(self) -> bool:
+ return inspect.isasyncgenfunction(self._fn)
def _validate_action_function(fn: Callable):
@@ -934,7 +942,6 @@ def _validate_action_function(fn: Callable):
)
return_type = sig.return_annotation
- print(return_type)
if (
return_type is inspect.Signature.empty
or not typing.get_origin(return_type) in {tuple, typing.Tuple}
@@ -1036,32 +1043,24 @@ def streaming_action(
# return the final result
return {'response': full_response}, state.update(response=full_response)
- TODO -- async
"""
- # TODO -- see if we want to consolidate this in some way with the above
- # Currently this is our only other way to use a decorator to implement actions, so it is OK to duplicate
- def decorator(fn) -> FunctionRepresentingAction:
- is_async = inspect.isasyncgenfunction(fn)
- if not is_async:
- setattr(
- fn,
- FunctionBasedAction.ACTION_FUNCTION,
- FunctionBasedStreamingAction(fn, reads, writes),
- )
- else:
- setattr(
- fn,
- FunctionBasedAction.ACTION_FUNCTION,
- FunctionBasedStreamingActionAsync(fn, reads, writes),
- )
+ def wrapped(fn) -> FunctionRepresentingAction:
+ setattr(
+ fn,
+ FunctionBasedAction.ACTION_FUNCTION,
+ FunctionBasedStreamingAction(fn, reads, writes),
+ )
setattr(fn, "bind", types.MethodType(bind, fn))
return fn
- return decorator
+ return wrapped
+
+
+ActionT = TypeVar("ActionT", bound=Action)
-def create_action(action_: Union[Callable, Action], name: str) -> Action:
+def create_action(action_: Union[Callable, ActionT], name: str) -> ActionT:
"""Factory function to create an action. This is meant to be called by
the ApplicationBuilder, and not by the user. The internal API may change.
diff --git a/burr/core/application.py b/burr/core/application.py
index 086814a7..36ee7fdc 100644
--- a/burr/core/application.py
+++ b/burr/core/application.py
@@ -32,7 +32,6 @@ from burr.core.action import (
Reducer,
SingleStepAction,
SingleStepStreamingAction,
- SingleStepStreamingActionAsync,
StreamingAction,
StreamingResultContainer,
create_action,
@@ -230,17 +229,26 @@ def _run_single_step_action(
def _run_single_step_streaming_action(
action: SingleStepStreamingAction, state: State, inputs: Optional[Dict[str, Any]]
-) -> Generator[dict, None, Tuple[dict, State]]:
+) -> Generator[Tuple[dict, Optional[State]], None, None]:
action.validate_inputs(inputs)
generator = action.stream_run_and_update(state, **inputs)
- result, state = yield from generator
+ result = None
+ state_update = None
+ for result, state_update in generator:
+ if state_update is None:
+ yield result, None
+ if state_update is None:
+ raise ValueError(
+ f"Action {action.name} did not return a state update. For streaming actions, the last yield "
+ f"statement must be a tuple of (result, state_update). For example, yield dict(foo='bar'), state.update(foo='bar')"
+ )
_validate_result(result, action.name)
- _validate_reducer_writes(action, state, action.name)
- return result, state
+ _validate_reducer_writes(action, state_update, action.name)
+ yield result, state_update
-async def _run_single_step_streaming_action_async(
- action: SingleStepStreamingActionAsync, state: State, inputs: Optional[Dict[str, Any]]
+async def _arun_single_step_streaming_action(
+ action: SingleStepStreamingAction, state: State, inputs: Optional[Dict[str, Any]]
) -> AsyncGenerator[Tuple[dict, Optional[State]], None]:
action.validate_inputs(inputs)
generator = action.stream_run_and_update(state, **inputs)
@@ -270,24 +278,39 @@ async def _run_single_step_streaming_action_async(
def _run_multi_step_streaming_action(
action: StreamingAction, state: State, inputs: Optional[Dict[str, Any]]
-) -> Generator[dict, None, Tuple[dict, State]]:
+) -> Generator[Tuple[dict, Optional[State]], None, None]:
action.validate_inputs(inputs)
generator = action.stream_run(state, **inputs)
- result = yield from generator
+ result = None
+ for item in generator:
+ # We want to peek ahead so we can return the last one
+ # This is slightly eager, but only in the case in which we
+ # are using a multi-step streaming action
+ next_result = result
+ result = item
+ if next_result is not None:
+ yield next_result, None
+ # yield item, None
+ state_update = _run_reducer(action, state, result, action.name)
_validate_result(result, action.name)
- new_state = _run_reducer(action, state, result, action.name)
- return result, _state_update(state, new_state)
+ _validate_reducer_writes(action, state_update, action.name)
+ yield result, state_update
-async def _run_multi_step_streaming_action_async(
+async def _arun_multi_step_streaming_action(
action: AsyncStreamingAction, state: State, inputs: Optional[Dict[str, Any]]
) -> AsyncGenerator[Tuple[dict, Optional[State]], None]:
action.validate_inputs(inputs)
generator = action.stream_run(state, **inputs)
result = None
async for item in generator:
+ # We want to peek ahead so we can return the last one
+ # This is slightly eager, but only in the case in which we
+ # are using a multi-step streaming action
+ next_result = result
result = item
- yield item, None
+ if next_result is not None:
+ yield next_result, None
state_update = _run_reducer(action, state, result, action.name)
_validate_result(result, action.name)
_validate_reducer_writes(action, state_update, action.name)
@@ -793,7 +816,8 @@ class Application:
:param halt_before: The list of actions to halt before execution of. It will halt on the first one. Note that
if this is met, the streaming result container will be empty (and return None) for the result, having an empty generator.
:param inputs: Inputs to the action -- this is if this action requires an input that is passed in from the outside world
- :return: A streaming result container, which is a generator that will yield results as they come in, as wel as cache/give you the final result, and update state accordingly.
+ :return: A streaming result container, which is a generator that will yield results as they come in, as well as cache/give you the final result,
+ and update state accordingly.
This is meant to be used with streaming actions -- :py:meth:`streaming_action <burr.core.action.streaming_action>`
or :py:class:`StreamingAction <burr.core.action.StreamingAction>` It returns a
@@ -836,10 +860,10 @@ class Application:
delta = chunk.choices[0].delta.content
buffer.append(delta)
# yield partial results
- yield {'response': delta}
+ yield {'response': delta}, None # indicate that we are not done by returning a `None` state!
full_response = ''.join(buffer)
# return the final result
- return {'response': full_response}, state.update(response=full_response)
+ yield {'response': full_response}, state.update(response=full_response)
To use streaming_result, you pass in names of streaming actions (such as the one above) to the halt_after
parameter:
@@ -1001,7 +1025,109 @@ class Application:
halt_before: list[str] = None,
inputs: Optional[Dict[str, Any]] = None,
) -> Tuple[Action, AsyncStreamingResultContainer]:
- """Placeholder for the async version of stream_result. This is not yet implemented."""
+ """Streams a result out in an asynchronous manner.
+
+ :param halt_after: The list of actions to halt after execution of. It will halt on the first one.
+ :param halt_before: The list of actions to halt before execution of. It will halt on the first one. Note that
+ if this is met, the streaming result container will be empty (and return None) for the result, having an empty generator.
+ :param inputs: Inputs to the action -- this is if this action requires an input that is passed in from the outside world
+ :return: An asynchronous :py:class:`AsyncStreamingResultContainer <burr.core.action.AsyncStreamingResultContainer>`, which is a generator that will yield results as they come in, as well as cache/give you the final result,
+ and update state accordingly.
+
+ This is meant to be used with streaming actions -- :py:meth:`streaming_action <burr.core.action.streaming_action>`
+ or :py:class:`StreamingAction <burr.core.action.StreamingAction>` It returns a
+ :py:class:`StreamingResultContainer <burr.core.action.StreamingResultContainer>`, which has two capabilities:
+
+ 1. It is a generator that streams out the intermediate results of the action
+ 2. It has an async ``.get()`` method that returns the final result of the action, and the final state.
+
+ If ``.get()`` is called before the generator is exhausted, it will block until the generator is exhausted.
+
+ While this container is meant to work with streaming actions, it can also be used with non-streaming actions. In this case,
+ the generator will be empty, and the ``.get()`` method will return the final result and state.
+
+ The rules for halt_before and halt_after are the same as for :py:meth:`iterate <burr.core.application.Application.iterate>`,
+ and :py:meth:`run <burr.core.application.Application.run>`. In this case, `halt_before` will indicate a *non* streaming action,
+ which will be empty. Thus ``halt_after`` takes precedence -- if it is met, the streaming result container will contain the result of the
+ halt_after condition.
+
+ The :py:class:`AsyncStreamingResultContainer <burr.core.action.StreamingResultContainer>` is meant as a convenience -- specifically this allows for
+ hooks, callbacks, etc... so you can take the control flow and still have state updated afterwards. Hooks/state update will be called after an exception
+ is thrown during streaming, or the stream is completed. Note that it is undefined behavior to attempt to execute another action while a stream is in progress.
+
+
+ To see how this works, let's take the following action (simplified as a single-node workflow) as an example:
+
+ .. code-block:: python
+
+ client = openai.AsyncClient()
+
+ @streaming_action(reads=[], writes=['response'])
+ async def streaming_response(state: State, prompt: str) -> Generator[dict, None, Tuple[dict, State]]:
+ response = client.chat.completions.create(
+ model='gpt-3.5-turbo',
+ messages=[{
+ 'role': 'user',
+ 'content': prompt
+ }],
+ temperature=0,
+ )
+ buffer = []
+ async for chunk in response: # use an async for loop
+ delta = chunk.choices[0].delta.content
+ buffer.append(delta)
+ # yield partial results
+ yield {'response': delta}, None # indicate that we are not done by returning a `None` state!
+ # make sure to join with the buffer!
+ full_response = ''.join(buffer)
+ # yield the final result at the end + the state update
+ yield {'response': full_response}, state.update(response=full_response)
+
+ To use streaming_result, you pass in names of streaming actions (such as the one above) to the halt_after
+ parameter:
+
+ .. code-block:: python
+
+ application = ApplicationBuilder().with_actions(streaming_response=streaming_response)...build()
+ prompt = "Count to 100, with a comma between each number and no newlines. E.g., 1, 2, 3, ..."
+ action, streaming_result = application.astream_result(halt_after='streaming_response', inputs={"prompt": prompt})
+ async for result in streaming_result:
+ print(result['response']) # one by one
+
+ result, state = await streaming_result.get()
+ print(result['response']) # all at once
+
+ Note that if you have multiple halt_after conditions, you can use the ``.action`` attribute to get the action that
+ was run.
+
+ .. code-block:: python
+
+ application = ApplicationBuilder().with_actions(
+ streaming_response=streaming_response,
+ error=error # another function that outputs an error, streaming
+ )...build()
+ prompt = "Count to 100, with a comma between each number and no newlines. E.g., 1, 2, 3, ..."
+ action, streaming_result = await application.astream_result(halt_after='streaming_response', inputs={"prompt": prompt})
+ color = "red" if action.name == "error" else "green"
+ for result in streaming_result:
+ print(format(result['response'], color)) # assumes that error and streaming_response both have the same output shape
+
+ .. code-block:: python
+
+ application = ApplicationBuilder().with_actions(
+ streaming_response=streaming_response,
+ error=non_streaming_error # a non-streaming function that outputs an error
+ )...build()
+ prompt = "Count to 100, with a comma between each number and no newlines. E.g., 1, 2, 3, ..."
+ action, streaming_result = await application.astream_result(halt_after='streaming_response', inputs={"prompt": prompt})
+ color = "red" if action.name == "error" else "green"
+ if action.name == "streaming_response": # can also use the ``.streaming`` attribute of action
+ async for result in streaming_result:
+ print(format(result['response'], color)) # assumes that error and streaming_response both have the same output shape
+ else:
+ result, state = await output.get()
+ print(format(result['response'], color))
+ """
halt_before, halt_after, inputs = self._clean_iterate_params(
halt_before, halt_after, inputs
)
@@ -1078,10 +1204,16 @@ class Application:
results=result, final_state=state
)
if next_action.single_step:
- next_action = cast(SingleStepStreamingActionAsync, next_action)
+ next_action = cast(SingleStepStreamingAction, next_action)
if not next_action.is_async():
- raise ValueError("TODO -- convert")
- generator = _run_single_step_streaming_action_async(
+ raise ValueError(
+ f"Action: {next_action.name} is not"
+ "an async action, but is marked as streaming. "
+ "Currently we do not support running synchronous "
+ "streaming actions with astream_result, although we plan to in the future. "
+ "For now, convert the action to async. Please open up an issue if you hit this. "
+ )
+ generator = _arun_single_step_streaming_action(
next_action, self._state, action_inputs
)
return next_action, AsyncStreamingResultContainer(
@@ -1089,9 +1221,15 @@ class Application:
)
else:
if not next_action.is_async():
- raise ValueError("TODO -- convert")
+ raise ValueError(
+ f"Action: {next_action.name} is not"
+ "an async action, but is marked as streaming. "
+ "Currently we do not support running synchronous "
+ "streaming actions with astream_result, although we plan to in the future. "
+ "For now, convert the action to async. Please open up an issue if you hit this. "
+ )
next_action = cast(AsyncStreamingAction, next_action)
- generator = _run_multi_step_streaming_action_async(
+ generator = _arun_multi_step_streaming_action(
next_action, self._state, action_inputs
)
except Exception as e:
@@ -1112,10 +1250,6 @@ class Application:
return next_action, AsyncStreamingResultContainer(
generator, self._state, process_result, callback
)
- # raise NotImplementedError(
- # "This has not yet been implemented! See the github issue: https://github.com/DAGWorks-Inc/burr/issues/64"
- # " for details. Please comment or vote to get it implemented quickly!"
- # )
@telemetry.capture_function_usage
def visualize(
diff --git a/docs/concepts/state-machine.rst b/docs/concepts/state-machine.rst
index ae84471e..621683db 100644
--- a/docs/concepts/state-machine.rst
+++ b/docs/concepts/state-machine.rst
@@ -36,7 +36,7 @@ This is shown in the example from :ref:`getting started <simpleexample>`
Running
-------
-There are three APIs for executing an application.
+There are four APIs for executing an application.
``step``/``astep``
------------------
@@ -135,6 +135,12 @@ In the async context, you can run ``arun``:
``run`` and ``arun`` largely have the same behavior as ``iterate`` and ``aiterate``.
+``stream_result``/``astream_result``
+------------------------------------
+
+These allow you to stream responses back. Note they work best with streaming actions.
+Read about them in the :ref:`streaming <streaming>` section.
+
----------
Inspection
----------
diff --git a/docs/concepts/streaming-actions.rst b/docs/concepts/streaming-actions.rst
index 81736cb7..28709345 100644
--- a/docs/concepts/streaming-actions.rst
+++ b/docs/concepts/streaming-actions.rst
@@ -1,27 +1,63 @@
=================
Streaming Actions
=================
-
-.. _streaming_actions:
+.. _streaming:
.. note::
Burr actions can stream results! This enables you to display results to the user as
tokens are streamed in.
+
Actions can be implemented as streaming results. This enables a lower time-to-first-token and a more interactive
interface in the case of AI applications or streaming in of metrics in a model-training application. Broadly,
this is a tool to enable quicker user interaction in longer running actions that require user focus.
+Like other actions, these can be implemented both as functions and as classes, and can use synchronous or asynchrounous APIs.
+
+They are used differently from regular actions -- the application wraps their result in a
+:py:class:`StreamingResultContainer <burr.core.action.StreamingResultContainer>`, or a
+:py:class:`AsyncStreamingResultContainerAsync <burr.core.action.AsyncStreamingResultContainer>` for async streaming actions.
+
+----------
+Definition
+----------
+.. _streaming_actions:
+
Streaming actions can be implemented as a class or a function, just like actions. However, they have a few additional rules:
-#. They give intermediate results to the user as they are generated in the form of a generator
+#. They give intermediate results to the user as they are produced in the form of a generator
#. They are responsible for determining the relationship between the intermediate results and the final result
-#. They have a ``return`` statement that handles result collection and state update
+#. They yield intermediate results to the framework, and the last yield is the final result (with everything joined). Don't forget the last yield!
+
+
+The function-based streaming action is fairly simple. Note that
+we yield a tuple of the result and the state update. The state update is ``None`` for intermediate results and
+the final state for the final result.
-If you're not familiar with ``return`` statements in generators, you can read about them `here <https://www.python.org/dev/peps/pep-0380/>`_.
-The high-level idea is that the ``return`` statement is used to raise a ``StopIteration`` exception with a value that is returned to the caller.
-The framework uses this value to update the state of the action and to collect the final result.
+.. code-block:: python
+
+ from burr.core.action import streaming_action
+
+ @streaming_action(reads=["prompt"], writes=["prompt"])
+ def streaming_chat_call(state: State, **run_kwargs) -> Generator[Tuple[dict, Optional[State], None, None]]:
+ client = openai.Client()
+ response = client.chat.completions.create(
+ model='gpt-3.5-turbo',
+ messages=[{
+ 'role': 'user',
+ 'content': state["prompt"]
+ }],
+ temperature=0,
+ stream=True,
+ )
+ buffer = []
+ for chunk in response:
+ delta = chunk.choices[0].delta.content
+ buffer.append(delta)
+ yield {'response': delta}, None # No state update on intermediate results
+ full_response = ''.join(buffer)
+ yield {'response': full_response}, state.append(response=full_response) # Update state on final results
A class-based streaming action might look like this:
@@ -30,7 +66,7 @@ A class-based streaming action might look like this:
from burr.core.action import StreamingAction
class StreamingChatCall(StreamingAction):
- def stream_run(self, state: State, **run_kwargs) -> Generator[dict, None, dict]:
+ def stream_run(self, state: State, **run_kwargs) -> Generator[dict, None, None]:
client = openai.Client()
response = client.chat.completions.create(
model='gpt-3.5-turbo',
@@ -47,7 +83,7 @@ A class-based streaming action might look like this:
buffer.append(delta)
yield {'response': delta}
full_response = ''.join(buffer)
- return {'response': full_response}
+ yield {'response': full_response}
@property
def reads(self) -> list[str]:
@@ -62,35 +98,128 @@ A class-based streaming action might look like this:
The logic is split between ``stream_run``, which is responsible for generating the intermediate results and
joining them into the final result, and update, which is responsible for collecting the final result and
-updating the state. The ``return`` statement in ``stream_run`` is used to return the final result to the framework,
-which is passed to ``update``.
+updating the state. The final ``yield`` statement in ``stream_run`` is used to return the final result to the framework,
+which is passed to ``update``. Note that the class-based variant separates out run/update into two methods, meaning
+that it only yields the ``result`` and not the state update. The function-based variant, above, combined the two.
+
+``async`` streaming actions are also supported. The corresponding function-based async looks like this:
+
+.. collapse:: <code>fn_based_async</code>
+
+ .. code-block:: python
+
+ from burr.core.action import streaming_action
+
+ @streaming_action(reads=["prompt"], writes=["prompt"])
+ async def streaming_chat_call(state: State, **run_kwargs) -> AsyncGenerator[Tuple[dict, Optional[State]], None]:
+ client = openai.AsyncClient()
+ response = await client.chat.completions.create(
+ model='gpt-3.5-turbo',
+ messages=[{
+ 'role': 'user',
+ 'content': state["prompt"]
+ }],
+ temperature=0,
+ stream=True,
+ )
+ buffer = []
+ async for chunk in response: # loop over in async
+ delta = chunk.choices[0].delta.content
+ buffer.append(delta)
+ yield {'response': delta}, None # No state update on intermediate results
+ full_response = ''.join(buffer)
+ yield {'response': full_response}, state.append(response=full_response) # Update state on final results
+
+The class-based async streaming action will look like this:
+
+.. collapse:: <code>class_based_async</code>
+
+ .. code-block:: python
+
+ from burr.core.action import StreamingAction
+
+ class StreamingChatCallAsync(StreamingAction):
+ async def stream_run(self, state: State, **run_kwargs) -> AsyncGenerator[dict, None]:
+ client = openai.Client()
+ response = await client.chat.completions.create(
+ model='gpt-3.5-turbo',
+ messages=[{
+ 'role': 'user',
+ 'content': state["prompt"]
+ }],
+ temperature=0,
+ stream=True,
+ )
+ buffer = []
+ async for chunk in response:
+ delta = chunk.choices[0].delta.content
+ buffer.append(delta)
+ yield {'response': delta}
+ full_response = ''.join(buffer)
+ yield {'response': full_response}
+
+ @property
+ def reads(self) -> list[str]:
+ return ["prompt"]
+
+ @property
+ def writes(self) -> list[str]:
+ return ["response"]
+
+ def update(self, result: dict, state: State) -> State:
+ return state.append(response=result["response"])
+
+-----
+Usage
+-----
+
+When you call out to :py:meth:`stream_result <burr.core.application.Application.stream_result>` (as well as its corresponding async implementation
+:py:meth:`astream_result <burr.core.application.Application.astream_result>` on a streaming action, you will get
+a :py:class:`StreamingResultContainer <burr.core.action.StreamingResultContainer>`, or a :py:class:`AsyncStreamingResultContainerAsync <burr.core.action.AsyncStreamingResultContainer>`
+object.
+
+This object is effectively a cached iterator. You can use it as follows:
+
+.. code-block:: python
+
+ action, streaming_result = application.stream_result(
+ halt_after='streaming_response', inputs={"prompt": prompt}
+ )
+ for result in streaming_result:
+ print(result) # one by one
-The function-based equivalent would look very similar:
+ result, state = streaming_result.get()
+ print(result) # get the result
.. code-block:: python
- from burr.core.action import streaming_action
+ action, async_streaming_result = await application.astream_result(
+ halt_after='streaming_response', inputs={"prompt": prompt}
+ )
+ async for result in async_streaming_result:
+ print(result) # one by one
- @streaming_action(reads=["prompt"], writes=["prompt"])
- def streaming_chat_call(state: State, **run_kwargs) -> Generator[dict, None, Tuple[dict, State]]:
- client = openai.Client()
- response = client.chat.completions.create(
- model='gpt-3.5-turbo',
- messages=[{
- 'role': 'user',
- 'content': state["prompt"]
- }],
- temperature=0,
- stream=True,
- )
- buffer = []
- for chunk in response:
- delta = chunk.choices[0].delta.content
- buffer.append(delta)
- yield {'response': delta}
- full_response = ''.join(buffer)
- return {'response': full_response}, state.append(response=full_response)
+ result, state = await async_streaming_result.get()
+ print(result) # all at once
+
+
+Thus you can run this in a web-service, a streamlit app, etc...
+
+--------------
+Considerations
+--------------
+
+All hooks/state update will be called once the iterator completes, or an exception interrupts the iterator
+and it has to be cleaned up. You can call ``.stream_result()`` or ``.astream_result()`` on non-streaming
+results, and it will return a ``StreamingResultContainer`` with an empty iterator that returns the result.
+If streaming items are run as intermediate nodes in the graph, they will be run as normal actions
+(effectively fully exhausted), and the result will be returned as a single item. Currently
+you cannot use synchronous streaming actions as asynchronous streaming actions, but we will likely be
+adding a bridge.
-As you can see above, we're doing the same thing, with a bit of syntactic sugar to combine the ``update`` and ``return`` statements.
+In version ``0.18.0`` we changed the synchronous method of streaming to
+be consistent with the asynchronous method. If you're using the old version, there are a few changes you'll have to make (for the function-based API):
-Currently ``Async`` streaming actions are not supported. We will be adding shortly -- stay tuned! Follow the `issue on github <https://github.com/DAGWorks-Inc/burr/issues/64>`_ for more information.
+1. The return type of the streaming action should be ``Generator[Tuple[dict, Optional[State], None, None]]`` instead of ``Generator[dict, None, Tuple[dict, State]]``.
+2. All intermediate results should be yielded as ``yield {'response': delta}, None`` instead of ``yield {'response': delta}``.
+3. The final result will be a ``yield`` instead of a ``return``
diff --git a/docs/getting_started/simple-example.rst b/docs/getting_started/simple-example.rst
index a9b583cb..905687c2 100644
--- a/docs/getting_started/simple-example.rst
+++ b/docs/getting_started/simple-example.rst
@@ -206,4 +206,5 @@ Now that we've built a basic application, we can do the following with only a fe
1. :ref:`Add conditional edges <transitions>` -- add a condition as a third item in the tuple to the ``with_transitions`` method.
2. :ref:`Persist state to a database + reload <state-persistence>` -- add a ``initialize_from`` line to the builder and select a pre-existing/implement a custom persistence method.
3. :ref:`Add monitoring to track application data <tracking>` -- leverage ``with_tracker`` to track to the Burr UI and visualize your application live.
-4. `Generate test cases from prior runs <https://github.com/DAGWorks-Inc/burr/tree/main/examples/test-case-creation>`_ -- use the ``burr-testburr-test-case create`` command to automatically generate test cases for your LLM app.
+4. :ref:`Stream results back <streaming>` -- minimize time to first token by streaming results back to the user.
+5. `Generate test cases from prior runs <https://github.com/DAGWorks-Inc/burr/tree/main/examples/test-case-creation>`_ -- use the ``burr-testburr-test-case create`` command to automatically generate test cases for your LLM app.
diff --git a/examples/streaming-overview/README.md b/examples/streaming-overview/README.md
index a6a4531a..f1c33bd2 100644
--- a/examples/streaming-overview/README.md
+++ b/examples/streaming-overview/README.md
@@ -10,7 +10,13 @@ but it is built slightly differently (for streaming purposes).
## How to use
Run `streamlit run streamlit_app.py` from the command line and you will see the chatbot in action.
-Open up the burr UI `burr` and you can track the chatbot
+Open up the burr UI `burr` and you can track the chatbot.
+
+## Async
+
+We also have an async version in [async_application.py](async_application.py)
+which demonstrates how to use streaming async. We have not hooked this up
+to a streamlit application yet, but that should be trivial.
## Notebook
The notebook also shows how things work. <a target="_blank" href="https://colab.research.google.com/github/dagworks-inc/burr/blob/main/examples/streaming-overview/notebook.ipynb">
diff --git a/examples/streaming-overview/application.py b/examples/streaming-overview/application.py
index dc531395..b8140771 100644
--- a/examples/streaming-overview/application.py
+++ b/examples/streaming-overview/application.py
@@ -1,4 +1,4 @@
-from typing import Optional, Tuple
+from typing import Generator, Optional, Tuple
import openai
@@ -73,7 +73,7 @@ def prompt_for_more(state: State) -> Tuple[dict, State]:
@streaming_action(reads=["prompt", "chat_history", "mode"], writes=["response"])
def chat_response(
state: State, prepend_prompt: str, model: str = "gpt-3.5-turbo"
-) -> Tuple[dict, State]:
+) -> Generator[Tuple[dict, Optional[State]], None, None]:
"""Streaming action, as we don't have the result immediately. This makes it more interactive"""
chat_history = state["chat_history"].copy()
chat_history[-1]["content"] = f"{prepend_prompt}: {chat_history[-1]['content']}"
@@ -96,12 +96,12 @@ def chat_response(
buffer.append(chunk_str)
yield {
"response": {"content": chunk_str, "type": MODES[state["mode"]], "role": "assistant"}
- }
+ }, None
result = {
"response": {"content": "".join(buffer), "type": MODES[state["mode"]], "role": "assistant"}
}
- return result, state.update(**result).append(chat_history=result["response"])
+ yield result, state.update(**result).append(chat_history=result["response"])
@action(reads=["prompt", "chat_history"], writes=["response"])
@@ -180,7 +180,7 @@ TERMINAL_ACTIONS = [
"unsafe_response",
]
if __name__ == "__main__":
- app = application(hooks=[])
+ app = application()
app.visualize(
output_file_path="statemachine", include_conditions=False, view=True, format="png"
)
diff --git a/examples/streaming-overview/async_application.py b/examples/streaming-overview/async_application.py
index 3c491b6f..62e01a65 100644
--- a/examples/streaming-overview/async_application.py
+++ b/examples/streaming-overview/async_application.py
@@ -195,7 +195,6 @@ async def it(app):
async def main():
app = application()
await it(app)
- await it(app)
if __name__ == "__main__":
diff --git a/tests/core/test_action.py b/tests/core/test_action.py
index ecf8348a..02747163 100644
--- a/tests/core/test_action.py
+++ b/tests/core/test_action.py
@@ -1,5 +1,5 @@
import asyncio
-from typing import AsyncGenerator, Generator, Optional, Tuple
+from typing import AsyncGenerator, Generator, Optional, Tuple, cast
import pytest
@@ -14,7 +14,6 @@ from burr.core.action import (
Result,
SingleStepAction,
SingleStepStreamingAction,
- SingleStepStreamingActionAsync,
StreamingAction,
StreamingResultContainer,
_validate_action_function,
@@ -205,8 +204,8 @@ def test_function_based_action_with_inputs():
res = state["input_variable"] + bound_input + unbound_input
return {"output_variable": res}, state.update(output_variable=res)
- fn_based_action: SingleStepAction = create_action(
- my_action.bind(bound_input=10), name="my_action"
+ fn_based_action = cast(
+ SingleStepAction, create_action(my_action.bind(bound_input=10), name="my_action")
)
assert fn_based_action.inputs == (["unbound_input"], [])
result, state = fn_based_action.run_and_update(State({"input_variable": 1}), unbound_input=100)
@@ -361,13 +360,15 @@ def test_create_action_fn_api_with_bind():
def test_create_action_streaming_fn_api_with_bind():
@streaming_action(reads=["input_variable"], writes=["output_variable"])
- def test_action(state: State, prefix: str) -> Generator[dict, None, Tuple[dict, State]]:
+ def test_action(
+ state: State, prefix: str
+ ) -> Generator[Tuple[dict, Optional[State]], None, None]:
buffer = []
for c in prefix + state["input_variable"]:
buffer.append(c)
- yield {"output_variable": c} # intermediate results
+ yield {"output_variable": c}, None # intermediate results
result = {"output_variable": "".join(buffer)}
- return result, state.update(output_variable=result["output_variable"])
+ yield result, state.update(output_variable=result["output_variable"])
created_action = create_action(test_action.bind(prefix="prefix_"), name="my_action")
assert created_action.streaming
@@ -377,16 +378,12 @@ def test_create_action_streaming_fn_api_with_bind():
assert created_action.writes == ["output_variable"]
assert created_action.single_step
gen = created_action.stream_run_and_update(State({"input_variable": "foo"}))
- out = []
- while True:
- try:
- out.append(next(gen)["output_variable"])
- except StopIteration as e:
- result, state = e.value
- assert result == {"output_variable": "prefix_foo"}
- assert state.get_all() == {"input_variable": "foo", "output_variable": "prefix_foo"}
- break
- assert out == list("prefix_foo")
+ out = [item for item in gen]
+ final_result, state = out[-1]
+ intermediate_results = [item[0]["output_variable"] for item in out[:-1]]
+ assert final_result == {"output_variable": "prefix_foo"}
+ assert state.get_all() == {"input_variable": "foo", "output_variable": "prefix_foo"}
+ assert intermediate_results == list("prefix_foo")
async def test_create_action_streaming_fn_api_with_bind_async():
@@ -405,7 +402,7 @@ async def test_create_action_streaming_fn_api_with_bind_async():
created_action = create_action(test_action.bind(prefix="prefix_"), name="my_action")
assert created_action.streaming # streaming
assert created_action.is_async() # async
- assert isinstance(created_action, SingleStepStreamingActionAsync)
+ assert isinstance(created_action, SingleStepStreamingAction)
assert created_action.name == "my_action"
assert created_action.reads == ["input_variable"]
assert created_action.writes == ["output_variable"]
@@ -462,12 +459,12 @@ def test_streaming_action_stream_run():
class SimpleStreamingAction(StreamingAction):
def stream_run(
self, state: State, **run_kwargs
- ) -> Generator[dict, None, Tuple[dict, State]]:
+ ) -> Generator[Tuple[dict, Optional[State]], None, None]:
buffer = []
for char in state["echo"]:
- yield {"response": char}
+ yield {"response": char}, None
buffer.append(char)
- return {"response": "".join(buffer)}, state.update(response="".join(buffer))
+ yield {"response": "".join(buffer)}, state.update(response="".join(buffer))
@property
def reads(self) -> list[str]:
@@ -489,15 +486,13 @@ def test_streaming_action_stream_run():
async def test_streaming_action_stream_run_async():
class SimpleAsyncStreamingAction(AsyncStreamingAction):
- async def stream_run(
- self, state: State, **run_kwargs
- ) -> AsyncGenerator[Tuple[dict, Optional[State]], None]:
+ async def stream_run(self, state: State, **run_kwargs) -> AsyncGenerator[dict, None]:
buffer = []
for char in state["echo"]:
yield {"response": char}, None
await asyncio.sleep(0.01)
buffer.append(char)
- yield {"response": "".join(buffer)}, state.update(response="".join(buffer))
+ yield {"response": "".join(buffer)}
@property
def reads(self) -> list[str]:
@@ -512,18 +507,19 @@ async def test_streaming_action_stream_run_async():
action = SimpleAsyncStreamingAction()
STR = "test streaming action"
- result, state_update = await action.run(State({"echo": STR}))
+ result = await action.run(State({"echo": STR}))
+ state_update = action.update(result, State({"echo": STR}))
assert result["response"] == STR
assert state_update["response"] == STR
-def sample_generator(chars: str) -> Generator[dict, None, Tuple[dict, State]]:
+def sample_generator(chars: str) -> Generator[Tuple[dict, Optional[State]], None, None]:
buffer = []
for c in chars:
buffer.append(c)
- yield {"response": c}
+ yield {"response": c}, None
joined = "".join(buffer)
- return {"response": joined}, State({"response": joined})
+ yield {"response": joined}, State({"response": joined})
def test_streaming_result_container_iterate():
@@ -588,8 +584,8 @@ def test_streaming_result_callback_error():
callback=lambda r, s, e: called.append((r, s, e)),
)
try:
- next(container)
- raise SentinelError("error")
+ for _ in container:
+ raise SentinelError("error")
finally:
assert len(called) == 1
((result, state, error),) = called
@@ -674,14 +670,13 @@ def test_streaming_result_callback_error_async():
"""
called = []
- async def test_fn():
- class SentinelError(Exception):
- pass
+ class SentinelError(Exception):
+ pass
+ async def test_fn():
try:
async def callback(r: Optional[dict], s: State, e: Exception):
- print("calling back")
called.append((r, s, e))
string_value = "test streaming action"
@@ -691,13 +686,8 @@ def test_streaming_result_callback_error_async():
process_result=lambda r, s: (r, s),
callback=callback,
)
- try:
- async for _ in container:
- raise SentinelError("error")
- finally:
- # pass
- print("asserting")
-
+ async for _ in container:
+ raise SentinelError("error")
# Exception is currently not exactly what we want, so won't assert on that.
# See note in StreamingResultContainer
except SentinelError:
diff --git a/tests/core/test_application.py b/tests/core/test_application.py
index 07e187fc..8924347f 100644
--- a/tests/core/test_application.py
+++ b/tests/core/test_application.py
@@ -9,13 +9,13 @@ import pytest
from burr.core import State
from burr.core.action import (
Action,
+ AsyncGenerator,
AsyncStreamingAction,
Condition,
Reducer,
Result,
SingleStepAction,
SingleStepStreamingAction,
- SingleStepStreamingActionAsync,
StreamingAction,
action,
default,
@@ -27,15 +27,15 @@ from burr.core.application import (
ApplicationBuilder,
Transition,
_arun_function,
+ _arun_multi_step_streaming_action,
_arun_single_step_action,
+ _arun_single_step_streaming_action,
_assert_set,
_run_function,
_run_multi_step_streaming_action,
- _run_multi_step_streaming_action_async,
_run_reducer,
_run_single_step_action,
_run_single_step_streaming_action,
- _run_single_step_streaming_action_async,
_validate_actions,
_validate_start,
_validate_transitions,
@@ -380,9 +380,9 @@ def test_run_single_step_streaming_action_errors_missing_write():
class BrokenAction(SingleStepStreamingAction):
def stream_run_and_update(
self, state: State, **run_kwargs
- ) -> Generator[dict, None, Tuple[dict, State]]:
- yield {}
- return {"present_value": 1}, state.update(present_value=1)
+ ) -> Generator[Tuple[dict, Optional[State]], None, None]:
+ yield {}, None
+ yield {"present_value": 1}, state.update(present_value=1)
@property
def reads(self) -> list[str]:
@@ -399,13 +399,13 @@ def test_run_single_step_streaming_action_errors_missing_write():
collections.deque(gen, maxlen=0) # exhaust the generator
-def test_run_single_step_streaming_action_errors_missing_write_async():
+async def test_run_single_step_streaming_action_errors_missing_write_async():
class BrokenAction(SingleStepStreamingAction):
- def stream_run_and_update(
+ async def stream_run_and_update(
self, state: State, **run_kwargs
- ) -> Generator[dict, None, Tuple[dict, State]]:
- yield {}
- return {"present_value": 1}, state.update(present_value=1)
+ ) -> AsyncGenerator[Tuple[dict, Optional[State]], None]:
+ yield {}, None
+ yield {"present_value": 1}, state.update(present_value=1)
@property
def reads(self) -> list[str]:
@@ -418,15 +418,15 @@ def test_run_single_step_streaming_action_errors_missing_write_async():
action = BrokenAction()
state = State()
with pytest.raises(ValueError, match="missing_value"):
- gen = _run_single_step_streaming_action(action, state, inputs={})
- collections.deque(gen, maxlen=0) # exhaust the generator
+ gen = _arun_single_step_streaming_action(action, state, inputs={})
+ [result async for result in gen] # exhaust the generator
def test_run_multi_step_streaming_action_errors_missing_write():
class BrokenAction(StreamingAction):
- def stream_run(self, state: State, **run_kwargs) -> Generator[dict, None, dict]:
+ def stream_run(self, state: State, **run_kwargs) -> Generator[dict, None, None]:
yield {}
- return {"present_value": 1}
+ yield {"present_value": 1}
def update(self, result: dict, state: State) -> State:
return state.update(present_value=1)
@@ -505,7 +505,7 @@ class SingleStepCounterWithInputsAsync(SingleStepCounterAsync):
class StreamingCounter(StreamingAction):
- def stream_run(self, state: State, **run_kwargs) -> Generator[dict, None, dict]:
+ def stream_run(self, state: State, **run_kwargs) -> Generator[dict, None, None]:
if "steps_per_count" in run_kwargs:
steps_per_count = run_kwargs["granularity"]
else:
@@ -513,7 +513,7 @@ class StreamingCounter(StreamingAction):
count = state["count"]
for i in range(steps_per_count):
yield {"count": count + ((i + 1) / 10)}
- return {"count": count + 1}
+ yield {"count": count + 1}
@property
def reads(self) -> list[str]:
@@ -528,7 +528,7 @@ class StreamingCounter(StreamingAction):
class AsyncStreamingCounter(AsyncStreamingAction):
- async def stream_run(self, state: State, **run_kwargs) -> typing.AsyncGenerator[dict, None]:
+ async def stream_run(self, state: State, **run_kwargs) -> AsyncGenerator[dict, None]:
if "steps_per_count" in run_kwargs:
steps_per_count = run_kwargs["granularity"]
else:
@@ -537,6 +537,8 @@ class AsyncStreamingCounter(AsyncStreamingAction):
for i in range(steps_per_count):
await asyncio.sleep(0.01)
yield {"count": count + (i + 1) / 10}
+ await asyncio.sleep(0.01)
+ yield {"count": count + 1}
@property
def reads(self) -> list[str]:
@@ -553,12 +555,12 @@ class AsyncStreamingCounter(AsyncStreamingAction):
class SingleStepStreamingCounter(SingleStepStreamingAction):
def stream_run_and_update(
self, state: State, **run_kwargs
- ) -> Generator[dict, None, Tuple[dict, State]]:
+ ) -> Generator[Tuple[dict, Optional[State]], None, None]:
steps_per_count = run_kwargs.get("granularity", 10)
count = state["count"]
for i in range(steps_per_count):
- yield {"count": count + ((i + 1) / 10)}
- return {"count": count + 1}, state.update(count=count + 1).append(tracker=count + 1)
+ yield {"count": count + ((i + 1) / 10)}, None
+ yield {"count": count + 1}, state.update(count=count + 1).append(tracker=count + 1)
@property
def reads(self) -> list[str]:
@@ -569,10 +571,10 @@ class SingleStepStreamingCounter(SingleStepStreamingAction):
return ["count", "tracker"]
-class SingleStepStreamingCounterAsync(SingleStepStreamingActionAsync):
+class SingleStepStreamingCounterAsync(SingleStepStreamingAction):
async def stream_run_and_update(
self, state: State, **run_kwargs
- ) -> typing.AsyncGenerator[Tuple[dict, Optional[State]], None]:
+ ) -> AsyncGenerator[Tuple[dict, Optional[State]], None]:
steps_per_count = run_kwargs.get("granularity", 10)
count = state["count"]
for i in range(steps_per_count):
@@ -593,7 +595,7 @@ class SingleStepStreamingCounterAsync(SingleStepStreamingActionAsync):
class StreamingActionIncorrectResultType(StreamingAction):
def stream_run(self, state: State, **run_kwargs) -> Generator[dict, None, dict]:
yield {}
- return "not a dict"
+ yield "not a dict"
@property
def reads(self) -> list[str]:
@@ -608,7 +610,7 @@ class StreamingActionIncorrectResultType(StreamingAction):
class StreamingActionIncorrectResultTypeAsync(AsyncStreamingAction):
- async def stream_run(self, state: State, **run_kwargs) -> typing.AsyncGenerator[dict, None]:
+ async def stream_run(self, state: State, **run_kwargs) -> AsyncGenerator[dict, None]:
yield {}
yield "not a dict"
@@ -627,9 +629,9 @@ class StreamingActionIncorrectResultTypeAsync(AsyncStreamingAction):
class StreamingSingleStepActionIncorrectResultType(SingleStepStreamingAction):
def stream_run_and_update(
self, state: State, **run_kwargs
- ) -> Generator[dict, None, Tuple[dict, State]]:
- yield {}
- return "not a dict", state
+ ) -> Generator[Tuple[dict, Optional[State]], None, None]:
+ yield {}, State
+ yield "not a dict", state
@property
def reads(self) -> list[str]:
@@ -640,7 +642,7 @@ class StreamingSingleStepActionIncorrectResultType(SingleStepStreamingAction):
return []
-class StreamingSingleStepActionIncorrectResultTypeAsync(SingleStepStreamingActionAsync):
+class StreamingSingleStepActionIncorrectResultTypeAsync(SingleStepStreamingAction):
async def stream_run_and_update(
self, state: State, **run_kwargs
) -> typing.AsyncGenerator[Tuple[dict, Optional[State]], None]:
@@ -758,21 +760,20 @@ def test__run_multistep_streaming_action():
state = State({"count": 0, "tracker": []})
generator = _run_multi_step_streaming_action(action, state, inputs={})
last_result = -1
- try:
- while True:
- result = next(generator)
+ result = None
+ for result, state in generator:
+ if last_result < 1:
+ # Otherwise you hit floating poit comparison problems
assert result["count"] > last_result
- last_result = result["count"]
- except StopIteration as e:
- result, state = e.value
- assert result == {"count": 1}
- assert state.subset("count", "tracker").get_all() == {"count": 1, "tracker": [1]}
+ last_result = result["count"]
+ assert result == {"count": 1}
+ assert state.subset("count", "tracker").get_all() == {"count": 1, "tracker": [1]}
async def test__run_multistep_streaming_action_async():
action = base_streaming_counter_async.with_name("counter")
state = State({"count": 0, "tracker": []})
- generator = _run_multi_step_streaming_action_async(action, state, inputs={})
+ generator = _arun_multi_step_streaming_action(action, state, inputs={})
last_result = -1
result = None
async for result, state in generator:
@@ -796,9 +797,9 @@ async def test__run_streaming_action_incorrect_result_type_async():
action = StreamingActionIncorrectResultTypeAsync()
state = State()
with pytest.raises(ValueError, match="returned a non-dict"):
- gen = _run_multi_step_streaming_action_async(action, state, inputs={})
- async for item in gen:
- print(item)
+ gen = _arun_multi_step_streaming_action(action, state, inputs={})
+ async for _ in gen:
+ pass
def test__run_single_step_streaming_action_incorrect_result_type():
@@ -813,7 +814,7 @@ async def test__run_single_step_streaming_action_incorrect_result_type_async():
action = StreamingSingleStepActionIncorrectResultTypeAsync()
state = State()
with pytest.raises(ValueError, match="returned a non-dict"):
- gen = _run_single_step_streaming_action_async(action, state, inputs={})
+ gen = _arun_single_step_streaming_action(action, state, inputs={})
_ = [item async for item in gen]
@@ -822,21 +823,21 @@ def test__run_single_step_streaming_action():
state = State({"count": 0, "tracker": []})
generator = _run_single_step_streaming_action(action, state, inputs={})
last_result = -1
- try:
- while True:
- result = next(generator)
+ result, state = None, None
+ for result, state in generator:
+ if last_result < 1:
+ # Otherwise you hit comparison issues
+ # This is because we get to the last one, which is the final result
assert result["count"] > last_result
- last_result = result["count"]
- except StopIteration as e:
- result, state = e.value
- assert result == {"count": 1}
- assert state.subset("count", "tracker").get_all() == {"count": 1, "tracker": [1]}
+ last_result = result["count"]
+ assert result == {"count": 1}
+ assert state.subset("count", "tracker").get_all() == {"count": 1, "tracker": [1]}
async def test__run_single_step_streaming_action_async():
async_action = base_streaming_single_step_counter_async.with_name("counter")
state = State({"count": 0, "tracker": []})
- generator = _run_single_step_streaming_action_async(async_action, state, inputs={})
+ generator = _arun_single_step_streaming_action(async_action, state, inputs={})
last_result = -1
result, state = None, None
async for result, state in generator:
@@ -1529,6 +1530,8 @@ async def test_astream_result_halt_after_run_through_streaming():
action_tracker = ActionTracker()
counter_action = base_streaming_single_step_counter_async.with_name("counter")
counter_action_2 = base_streaming_single_step_counter_async.with_name("counter_2")
+ assert counter_action.is_async()
+ assert counter_action_2.is_async()
app = Application(
actions=[counter_action, counter_action_2],
transitions=[