Skip to content

armgit / dspy

Public Imported from GitHub
0

DSPy: The framework for programming—not prompting—language models

Python

support token streaming with json adapter (#8158)

Chen Qianb7375196ba95
diff --git a/dspy/adapters/json_adapter.py b/dspy/adapters/json_adapter.py
index 605daf148..4c6c86861 100644
--- a/dspy/adapters/json_adapter.py
+++ b/dspy/adapters/json_adapter.py
@@ -17,7 +17,6 @@ from dspy.adapters.utils import (
translate_field_type,
)
from dspy.clients.lm import LM
-from dspy.dsp.utils.settings import settings
from dspy.signatures.signature import Signature, SignatureMeta
logger = logging.getLogger(__name__)
@@ -48,10 +47,6 @@ class JSONAdapter(ChatAdapter):
provider = lm.model.split("/", 1)[0] or "openai"
params = litellm.get_supported_openai_params(model=lm.model, custom_llm_provider=provider)
- stream_listeners = settings.stream_listeners or []
- if len(stream_listeners) > 0:
- raise ValueError("Stream listener is not yet supported for JsonAdapter, please use ChatAdapter instead.")
-
# If response_format is not supported, use basic call
if not params or "response_format" not in params:
return super().__call__(lm, lm_kwargs, signature, demos, inputs)
diff --git a/dspy/streaming/streaming_listener.py b/dspy/streaming/streaming_listener.py
index 1efdf6213..67370dd8c 100644
--- a/dspy/streaming/streaming_listener.py
+++ b/dspy/streaming/streaming_listener.py
@@ -5,6 +5,9 @@ from typing import TYPE_CHECKING, Any, List
from litellm import ModelResponseStream
+from dspy.adapters.chat_adapter import ChatAdapter
+from dspy.adapters.json_adapter import JSONAdapter
+from dspy.dsp.utils.settings import settings
from dspy.streaming.messages import StreamResponse
if TYPE_CHECKING:
@@ -33,10 +36,29 @@ class StreamListener:
self.stream_end = False
self.cache_hit = False
- self.start_identifier = f"[[ ## {self.signature_field_name} ## ]]"
- self.end_identifier = re.compile(r"\[\[ ## (\w+) ## \]\]")
+ self.json_adapter_start_identifier = f'{{"{self.signature_field_name}":"' # noqa: Q000
+ self.json_adapter_end_identifier = re.compile(r"\w*\",\w*")
+
+ self.chat_adapter_start_identifier = f"[[ ## {self.signature_field_name} ## ]]"
+ self.chat_adapter_end_identifier = re.compile(r"\[\[ ## (\w+) ## \]\]")
def receive(self, chunk: ModelResponseStream):
+ if isinstance(settings.adapter, JSONAdapter):
+ start_identifier = self.json_adapter_start_identifier
+ end_identifier = self.json_adapter_end_identifier
+
+ start_indicator = "{"
+ elif isinstance(settings.adapter, ChatAdapter) or settings.adapter is None:
+ start_identifier = self.chat_adapter_start_identifier
+ end_identifier = self.chat_adapter_end_identifier
+
+ start_indicator = "[["
+ else:
+ raise ValueError(
+ f"Unsupported adapter for streaming: {settings.adapter}, please use either ChatAdapter or "
+ "JSONAdapter for streaming purposes."
+ )
+
if self.stream_end:
return
@@ -47,7 +69,7 @@ class StreamListener:
except Exception:
return
- if chunk_message and self.start_identifier in chunk_message:
+ if chunk_message and start_identifier in chunk_message:
# If the cache is hit, the chunk_message could be the full response. When it happens we can
# directly end the stream listening.
self.cache_hit = True
@@ -55,10 +77,10 @@ class StreamListener:
self.stream_end = True
return
- if len(self.field_start_queue) == 0 and "[[" in chunk_message:
- # We look for the pattern "[[ ## {self.signature_field_name} ## ]]" to identify the start
- # of the stream of our target field. Once "[[" is found, we start checking the next tokens
- # to see if they match our expected identifier.
+ if len(self.field_start_queue) == 0 and start_indicator in chunk_message:
+ # We look for the pattern of start_identifier, i.e., "[[ ## {self.signature_field_name} ## ]]" for
+ # ChatAdapter to identify the start of the stream of our target field. Once the start_indicator, i.e., "[["
+ # for ChatAdapter, is found, we start checking the next tokens
self.field_start_queue.append(chunk_message)
return
@@ -67,12 +89,12 @@ class StreamListener:
# tokens no longer match our expected identifier.
self.field_start_queue.append(chunk_message)
concat_message = "".join(self.field_start_queue).strip()
- start_token_index = concat_message.find("[[")
+ start_token_index = concat_message.find(start_indicator)
concat_message = concat_message[start_token_index:]
- if self.start_identifier == concat_message:
+ if start_identifier == concat_message:
# We have a full identifier, we can start the stream.
self.stream_start = True
- elif self.start_identifier.startswith(concat_message):
+ elif start_identifier.startswith(concat_message):
# The concanated tokens still match our expected identifier, we keep listening.
return
else:
@@ -83,12 +105,12 @@ class StreamListener:
token = None
self.field_end_queue.put(chunk_message)
if self.field_end_queue.qsize() > 10:
- # We keep the last 10 tokens in the buffer to check if they form a valid identifier for
- # "[[ ## {next_field_name} ## ]]" to identify the end of the current field. In most cases
- # 10 tokens are enough to cover "[[ ## {next_field_name} ## ]]".
+ # We keep the last 10 tokens in the buffer to check if they form a valid identifier for end_identifier,
+ # i.e., "[[ ## {next_field_name} ## ]]" for ChatAdapter to identify the end of the current field.
+ # In most cases 10 tokens are enough to cover the end_identifier for all adapters.
token = self.field_end_queue.get()
concat_message = "".join(self.field_end_queue.queue).strip()
- if re.search(self.end_identifier, concat_message):
+ if re.search(end_identifier, concat_message):
# The next field is identified, we can end the stream and flush out all tokens in the buffer.
self.stream_end = True
last_token = self.flush()
@@ -103,12 +125,21 @@ class StreamListener:
This method is called to flush out the last a few tokens when the stream is ended. These tokens
are in the buffer because we don't directly yield the tokens received by the stream listener
- with the purpose to not yield the tokens "[[ ## ... ## ]]".
+ with the purpose to not yield the end_identifier tokens, e.g., "[[ ## ... ## ]]" for ChatAdapter.
"""
last_tokens = "".join(self.field_end_queue.queue)
self.field_end_queue = Queue()
- start_token_index = last_tokens.find("[[")
- return last_tokens[:start_token_index]
+ if isinstance(settings.adapter, JSONAdapter):
+ boundary_index = last_tokens.find('",') # noqa: Q000
+ return last_tokens[:boundary_index]
+ elif isinstance(settings.adapter, ChatAdapter) or settings.adapter is None:
+ boundary_index = last_tokens.find("[[")
+ return last_tokens[:boundary_index]
+ else:
+ raise ValueError(
+ f"Unsupported adapter for streaming: {settings.adapter}, please use either ChatAdapter or "
+ "JSONAdapter for streaming purposes."
+ )
def find_predictor_for_stream_listeners(program: "Module", stream_listeners: List[StreamListener]):
diff --git a/tests/streaming/test_streaming.py b/tests/streaming/test_streaming.py
index 3cf2cfadd..9daf559d2 100644
--- a/tests/streaming/test_streaming.py
+++ b/tests/streaming/test_streaming.py
@@ -13,6 +13,7 @@ async def test_streamify_yields_expected_response_chunks(litellm_test_server):
model="openai/dspy-test-model",
api_base=api_base,
api_key="fakekey",
+ cache=True,
)
with dspy.context(lm=lm, adapter=dspy.JSONAdapter()):
@@ -45,6 +46,7 @@ async def test_streaming_response_yields_expected_response_chunks(litellm_test_s
model="openai/dspy-test-model",
api_base=api_base,
api_key="fakekey",
+ cache=False,
)
with dspy.context(lm=lm):
@@ -127,7 +129,7 @@ async def test_custom_status_streaming():
@pytest.mark.skipif(not os.getenv("OPENAI_API_KEY"), reason="OpenAI API key not found in environment variables")
@pytest.mark.anyio
-async def test_stream_listener():
+async def test_stream_listener_chat_adapter():
class MyProgram(dspy.Module):
def __init__(self):
self.predict1 = dspy.Predict("question->answer")
@@ -190,23 +192,22 @@ async def test_default_status_streaming_in_async_program():
@pytest.mark.skipif(not os.getenv("OPENAI_API_KEY"), reason="OpenAI API key not found in environment variables")
@pytest.mark.anyio
-async def test_stream_listener_in_async_program():
+async def test_stream_listener_json_adapter():
class MyProgram(dspy.Module):
def __init__(self):
self.predict1 = dspy.Predict("question->answer")
self.predict2 = dspy.Predict("question, answer->judgement")
- async def acall(self, x: str, **kwargs):
- answer = await self.predict1.acall(question=x, **kwargs)
- judgement = await self.predict2.acall(question=x, answer=answer, **kwargs)
+ def __call__(self, x: str, **kwargs):
+ answer = self.predict1(question=x, **kwargs)
+ judgement = self.predict2(question=x, answer=answer, **kwargs)
return judgement
# Turn off the cache to ensure the stream is produced.
- dspy.settings.configure(lm=dspy.LM("openai/gpt-4o-mini", cache=False))
+ dspy.settings.configure(lm=dspy.LM("openai/gpt-4o-mini", cache=False), adapter=dspy.JSONAdapter())
my_program = MyProgram()
program = dspy.streamify(
my_program,
- is_async_program=True,
stream_listeners=[
dspy.streaming.StreamListener(signature_field_name="answer"),
dspy.streaming.StreamListener(signature_field_name="judgement"),