DSPy: The framework for programming—not prompting—language models
Python
support token streaming with json adapter (#8158)
Chen Qianb7375196ba95
diff --git a/dspy/adapters/json_adapter.py b/dspy/adapters/json_adapter.pyindex 605daf148..4c6c86861 100644--- a/dspy/adapters/json_adapter.py+++ b/dspy/adapters/json_adapter.py@@ -17,7 +17,6 @@ from dspy.adapters.utils import (translate_field_type,)from dspy.clients.lm import LM-from dspy.dsp.utils.settings import settingsfrom dspy.signatures.signature import Signature, SignatureMetalogger = logging.getLogger(__name__)@@ -48,10 +47,6 @@ class JSONAdapter(ChatAdapter):provider = lm.model.split("/", 1)[0] or "openai"params = litellm.get_supported_openai_params(model=lm.model, custom_llm_provider=provider)- stream_listeners = settings.stream_listeners or []- if len(stream_listeners) > 0:- raise ValueError("Stream listener is not yet supported for JsonAdapter, please use ChatAdapter instead.")-# If response_format is not supported, use basic callif not params or "response_format" not in params:return super().__call__(lm, lm_kwargs, signature, demos, inputs)diff --git a/dspy/streaming/streaming_listener.py b/dspy/streaming/streaming_listener.pyindex 1efdf6213..67370dd8c 100644--- a/dspy/streaming/streaming_listener.py+++ b/dspy/streaming/streaming_listener.py@@ -5,6 +5,9 @@ from typing import TYPE_CHECKING, Any, Listfrom litellm import ModelResponseStream+from dspy.adapters.chat_adapter import ChatAdapter+from dspy.adapters.json_adapter import JSONAdapter+from dspy.dsp.utils.settings import settingsfrom dspy.streaming.messages import StreamResponseif TYPE_CHECKING:@@ -33,10 +36,29 @@ class StreamListener:self.stream_end = Falseself.cache_hit = False- self.start_identifier = f"[[ ## {self.signature_field_name} ## ]]"- self.end_identifier = re.compile(r"\[\[ ## (\w+) ## \]\]")+ self.json_adapter_start_identifier = f'{{"{self.signature_field_name}":"' # noqa: Q000+ self.json_adapter_end_identifier = re.compile(r"\w*\",\w*")++ self.chat_adapter_start_identifier = f"[[ ## {self.signature_field_name} ## ]]"+ self.chat_adapter_end_identifier = re.compile(r"\[\[ ## (\w+) ## \]\]")def receive(self, chunk: ModelResponseStream):+ if isinstance(settings.adapter, JSONAdapter):+ start_identifier = self.json_adapter_start_identifier+ end_identifier = self.json_adapter_end_identifier++ start_indicator = "{"+ elif isinstance(settings.adapter, ChatAdapter) or settings.adapter is None:+ start_identifier = self.chat_adapter_start_identifier+ end_identifier = self.chat_adapter_end_identifier++ start_indicator = "[["+ else:+ raise ValueError(+ f"Unsupported adapter for streaming: {settings.adapter}, please use either ChatAdapter or "+ "JSONAdapter for streaming purposes."+ )+if self.stream_end:return@@ -47,7 +69,7 @@ class StreamListener:except Exception:return- if chunk_message and self.start_identifier in chunk_message:+ if chunk_message and start_identifier in chunk_message:# If the cache is hit, the chunk_message could be the full response. When it happens we can# directly end the stream listening.self.cache_hit = True@@ -55,10 +77,10 @@ class StreamListener:self.stream_end = Truereturn- if len(self.field_start_queue) == 0 and "[[" in chunk_message:- # We look for the pattern "[[ ## {self.signature_field_name} ## ]]" to identify the start- # of the stream of our target field. Once "[[" is found, we start checking the next tokens- # to see if they match our expected identifier.+ if len(self.field_start_queue) == 0 and start_indicator in chunk_message:+ # We look for the pattern of start_identifier, i.e., "[[ ## {self.signature_field_name} ## ]]" for+ # ChatAdapter to identify the start of the stream of our target field. Once the start_indicator, i.e., "[["+ # for ChatAdapter, is found, we start checking the next tokensself.field_start_queue.append(chunk_message)return@@ -67,12 +89,12 @@ class StreamListener:# tokens no longer match our expected identifier.self.field_start_queue.append(chunk_message)concat_message = "".join(self.field_start_queue).strip()- start_token_index = concat_message.find("[[")+ start_token_index = concat_message.find(start_indicator)concat_message = concat_message[start_token_index:]- if self.start_identifier == concat_message:+ if start_identifier == concat_message:# We have a full identifier, we can start the stream.self.stream_start = True- elif self.start_identifier.startswith(concat_message):+ elif start_identifier.startswith(concat_message):# The concanated tokens still match our expected identifier, we keep listening.returnelse:@@ -83,12 +105,12 @@ class StreamListener:token = Noneself.field_end_queue.put(chunk_message)if self.field_end_queue.qsize() > 10:- # We keep the last 10 tokens in the buffer to check if they form a valid identifier for- # "[[ ## {next_field_name} ## ]]" to identify the end of the current field. In most cases- # 10 tokens are enough to cover "[[ ## {next_field_name} ## ]]".+ # We keep the last 10 tokens in the buffer to check if they form a valid identifier for end_identifier,+ # i.e., "[[ ## {next_field_name} ## ]]" for ChatAdapter to identify the end of the current field.+ # In most cases 10 tokens are enough to cover the end_identifier for all adapters.token = self.field_end_queue.get()concat_message = "".join(self.field_end_queue.queue).strip()- if re.search(self.end_identifier, concat_message):+ if re.search(end_identifier, concat_message):# The next field is identified, we can end the stream and flush out all tokens in the buffer.self.stream_end = Truelast_token = self.flush()@@ -103,12 +125,21 @@ class StreamListener:This method is called to flush out the last a few tokens when the stream is ended. These tokensare in the buffer because we don't directly yield the tokens received by the stream listener- with the purpose to not yield the tokens "[[ ## ... ## ]]".+ with the purpose to not yield the end_identifier tokens, e.g., "[[ ## ... ## ]]" for ChatAdapter."""last_tokens = "".join(self.field_end_queue.queue)self.field_end_queue = Queue()- start_token_index = last_tokens.find("[[")- return last_tokens[:start_token_index]+ if isinstance(settings.adapter, JSONAdapter):+ boundary_index = last_tokens.find('",') # noqa: Q000+ return last_tokens[:boundary_index]+ elif isinstance(settings.adapter, ChatAdapter) or settings.adapter is None:+ boundary_index = last_tokens.find("[[")+ return last_tokens[:boundary_index]+ else:+ raise ValueError(+ f"Unsupported adapter for streaming: {settings.adapter}, please use either ChatAdapter or "+ "JSONAdapter for streaming purposes."+ )def find_predictor_for_stream_listeners(program: "Module", stream_listeners: List[StreamListener]):diff --git a/tests/streaming/test_streaming.py b/tests/streaming/test_streaming.pyindex 3cf2cfadd..9daf559d2 100644--- a/tests/streaming/test_streaming.py+++ b/tests/streaming/test_streaming.py@@ -13,6 +13,7 @@ async def test_streamify_yields_expected_response_chunks(litellm_test_server):model="openai/dspy-test-model",api_base=api_base,api_key="fakekey",+ cache=True,)with dspy.context(lm=lm, adapter=dspy.JSONAdapter()):@@ -45,6 +46,7 @@ async def test_streaming_response_yields_expected_response_chunks(litellm_test_smodel="openai/dspy-test-model",api_base=api_base,api_key="fakekey",+ cache=False,)with dspy.context(lm=lm):@@ -127,7 +129,7 @@ async def test_custom_status_streaming():@pytest.mark.skipif(not os.getenv("OPENAI_API_KEY"), reason="OpenAI API key not found in environment variables")@pytest.mark.anyio-async def test_stream_listener():+async def test_stream_listener_chat_adapter():class MyProgram(dspy.Module):def __init__(self):self.predict1 = dspy.Predict("question->answer")@@ -190,23 +192,22 @@ async def test_default_status_streaming_in_async_program():@pytest.mark.skipif(not os.getenv("OPENAI_API_KEY"), reason="OpenAI API key not found in environment variables")@pytest.mark.anyio-async def test_stream_listener_in_async_program():+async def test_stream_listener_json_adapter():class MyProgram(dspy.Module):def __init__(self):self.predict1 = dspy.Predict("question->answer")self.predict2 = dspy.Predict("question, answer->judgement")- async def acall(self, x: str, **kwargs):- answer = await self.predict1.acall(question=x, **kwargs)- judgement = await self.predict2.acall(question=x, answer=answer, **kwargs)+ def __call__(self, x: str, **kwargs):+ answer = self.predict1(question=x, **kwargs)+ judgement = self.predict2(question=x, answer=answer, **kwargs)return judgement# Turn off the cache to ensure the stream is produced.- dspy.settings.configure(lm=dspy.LM("openai/gpt-4o-mini", cache=False))+ dspy.settings.configure(lm=dspy.LM("openai/gpt-4o-mini", cache=False), adapter=dspy.JSONAdapter())my_program = MyProgram()program = dspy.streamify(my_program,- is_async_program=True,stream_listeners=[dspy.streaming.StreamListener(signature_field_name="answer"),dspy.streaming.StreamListener(signature_field_name="judgement"),