Skip to content

armgit / dspy

Public Imported from GitHub
0

DSPy: The framework for programming—not prompting—language models

Python

fix bedrock streaming token tracking (#8431)

Benjamin Tripp77331de9dc4f
diff --git a/dspy/clients/lm.py b/dspy/clients/lm.py
index e9dfc901a..228d78b63 100644
--- a/dspy/clients/lm.py
+++ b/dspy/clients/lm.py
@@ -269,6 +269,9 @@ def _get_stream_completion_fn(
stream = cast(MemoryObjectSendStream, stream)
caller_predict_id = id(caller_predict) if caller_predict else None
+ if dspy.settings.track_usage:
+ request["stream_options"] = {"include_usage": True}
+
async def stream_completion(request: Dict[str, Any], cache_kwargs: Dict[str, Any]):
response = await litellm.acompletion(
cache=cache_kwargs,