Skip to content

Same agent Smaller bill. The tools and infrastructure to get more work from your AI. Fewer tokens. Less compute. Lower cost.

fix(python-sdk): capture streamed tool calls (#1542)

Mohd Quamar Tyagid921e53062d5
diff --git a/sdk/python/src/openlit/instrumentation/ai21/utils.py b/sdk/python/src/openlit/instrumentation/ai21/utils.py
index 59a263ac..24ef06ae 100644
--- a/sdk/python/src/openlit/instrumentation/ai21/utils.py
+++ b/sdk/python/src/openlit/instrumentation/ai21/utils.py
@@ -285,15 +285,37 @@ def process_chunk(scope, chunk):
chunked = response_as_dict(chunk)
# Collect message IDs and aggregated response from events
- if (
- len(chunked.get("choices", [])) > 0
- and "delta" in chunked.get("choices")[0]
- and "content" in chunked.get("choices")[0].get("delta", {})
- ):
- content = chunked.get("choices")[0].get("delta").get("content")
+ choices = chunked.get("choices", [])
+ if choices and "delta" in choices[0]:
+ delta = choices[0].get("delta", {})
+ content = delta.get("content")
if content:
scope._llmresponse += content
+ delta_tools = delta.get("tool_calls")
+ if delta_tools:
+ scope._tools = scope._tools or []
+ for tool in delta_tools:
+ index = tool.get("index", 0)
+ scope._tools.extend([{}] * (index + 1 - len(scope._tools)))
+ function = tool.get("function") or {}
+ if tool.get("id"):
+ scope._tools[index] = {
+ "id": tool["id"],
+ "function": {
+ # `or ""` handles explicit None from OpenAI-compatible SDKs
+ "name": function.get("name") or "",
+ "arguments": function.get("arguments") or "",
+ },
+ "type": tool.get("type", "function"),
+ }
+ elif scope._tools[index] and "function" in tool:
+ new_args = function.get("arguments") or ""
+ if scope._tools[index]["function"]["arguments"] is None:
+ scope._tools[index]["function"]["arguments"] = new_args
+ else:
+ scope._tools[index]["function"]["arguments"] += new_args
+
# Handle token usage including reasoning tokens and cached tokens
if chunked.get("usage"):
usage = chunked.get("usage", {})
diff --git a/sdk/python/src/openlit/instrumentation/groq/utils.py b/sdk/python/src/openlit/instrumentation/groq/utils.py
index a3ed33a2..739c8868 100644
--- a/sdk/python/src/openlit/instrumentation/groq/utils.py
+++ b/sdk/python/src/openlit/instrumentation/groq/utils.py
@@ -287,6 +287,30 @@ def process_chunk(scope, chunk):
scope._llmresponse += content
append_scope_reasoning(scope, extract_reasoning_content(delta, "reasoning"))
+ delta_tools = delta.get("tool_calls")
+ if delta_tools:
+ scope._tools = scope._tools or []
+ for tool in delta_tools:
+ index = tool.get("index", 0)
+ scope._tools.extend([{}] * (index + 1 - len(scope._tools)))
+ function = tool.get("function") or {}
+ if tool.get("id"):
+ scope._tools[index] = {
+ "id": tool["id"],
+ "function": {
+ # `or ""` handles explicit None from OpenAI-compatible SDKs
+ "name": function.get("name") or "",
+ "arguments": function.get("arguments") or "",
+ },
+ "type": tool.get("type", "function"),
+ }
+ elif scope._tools[index] and "function" in tool:
+ new_args = function.get("arguments") or ""
+ if scope._tools[index]["function"]["arguments"] is None:
+ scope._tools[index]["function"]["arguments"] = new_args
+ else:
+ scope._tools[index]["function"]["arguments"] += new_args
+
if chunked.get("x_groq") is not None:
if chunked.get("x_groq").get("usage") is not None:
# Handle token usage including reasoning tokens and cached tokens
diff --git a/sdk/python/src/openlit/instrumentation/together/utils.py b/sdk/python/src/openlit/instrumentation/together/utils.py
index aeade76c..848d55c7 100644
--- a/sdk/python/src/openlit/instrumentation/together/utils.py
+++ b/sdk/python/src/openlit/instrumentation/together/utils.py
@@ -66,14 +66,37 @@ def process_chunk(scope, chunk):
chunked = response_as_dict(chunk)
# Collect message IDs and aggregated response from events
- if len(chunked.get("choices")) > 0 and (
- "delta" in chunked.get("choices")[0]
- and "content" in chunked.get("choices")[0].get("delta")
- ):
- content = chunked.get("choices")[0].get("delta").get("content")
+ choices = chunked.get("choices", [])
+ if choices and "delta" in choices[0]:
+ delta = choices[0].get("delta", {})
+ content = delta.get("content")
if content:
scope._llmresponse += content
+ delta_tools = delta.get("tool_calls")
+ if delta_tools:
+ scope._tools = scope._tools or []
+ for tool in delta_tools:
+ index = tool.get("index", 0)
+ scope._tools.extend([{}] * (index + 1 - len(scope._tools)))
+ function = tool.get("function") or {}
+ if tool.get("id"):
+ scope._tools[index] = {
+ "id": tool["id"],
+ "function": {
+ # `or ""` handles explicit None from OpenAI-compatible SDKs
+ "name": function.get("name") or "",
+ "arguments": function.get("arguments") or "",
+ },
+ "type": tool.get("type", "function"),
+ }
+ elif scope._tools[index] and "function" in tool:
+ new_args = function.get("arguments") or ""
+ if scope._tools[index]["function"]["arguments"] is None:
+ scope._tools[index]["function"]["arguments"] = new_args
+ else:
+ scope._tools[index]["function"]["arguments"] += new_args
+
if chunked.get("usage"):
scope._response_id = chunked.get("id")
scope._response_model = chunked.get("model")
diff --git a/sdk/python/tests/test_streaming_tool_calls.py b/sdk/python/tests/test_streaming_tool_calls.py
new file mode 100644
index 00000000..bf64a3b9
--- /dev/null
+++ b/sdk/python/tests/test_streaming_tool_calls.py
@@ -0,0 +1,194 @@
+# pylint: disable=missing-function-docstring
+"""Regression tests: streamed tool-call deltas must populate scope._tools.
+
+Groq, AI21, and Together only assigned scope._tools on the non-streaming path.
+process_chunk ignored delta.tool_calls (and AI21/Together skipped tool-only
+deltas that had no content), so common_chat_logic never emitted tool span
+attributes. These tests drive the real process_chunk helpers with synthetic
+OpenAI-compatible chunks.
+
+Fixes: https://github.com/openlit/openlit/issues/1540
+"""
+
+from types import SimpleNamespace
+
+import pytest
+
+from openlit.instrumentation.ai21 import utils as ai21_utils
+from openlit.instrumentation.groq import utils as groq_utils
+from openlit.instrumentation.together import utils as together_utils
+
+PROVIDERS = [groq_utils, ai21_utils, together_utils]
+
+
+def _scope():
+ return SimpleNamespace(
+ _timestamps=[],
+ _start_time=0,
+ _llmresponse="",
+ _tools=None,
+ )
+
+
[email protected]("utils", PROVIDERS)
+def test_streaming_tool_calls_are_accumulated(utils):
+ scope = _scope()
+
+ utils.process_chunk(
+ scope,
+ {
+ "choices": [
+ {
+ "delta": {
+ "tool_calls": [
+ {
+ "index": 0,
+ "id": "call_1",
+ "type": "function",
+ "function": {
+ "name": "lookup",
+ "arguments": '{"q":"',
+ },
+ }
+ ]
+ }
+ }
+ ]
+ },
+ )
+ utils.process_chunk(
+ scope,
+ {
+ "choices": [
+ {
+ "delta": {
+ "tool_calls": [
+ {
+ "index": 0,
+ "function": {"arguments": "weather"},
+ }
+ ]
+ }
+ }
+ ]
+ },
+ )
+
+ assert scope._tools == [
+ {
+ "id": "call_1",
+ "type": "function",
+ "function": {"name": "lookup", "arguments": '{"q":"weather'},
+ }
+ ]
+
+
[email protected]("utils", PROVIDERS)
+def test_streaming_tool_calls_tolerate_null_arguments(utils):
+ """OpenAI-compatible SDKs often send arguments=None on the first delta."""
+ scope = _scope()
+
+ utils.process_chunk(
+ scope,
+ {
+ "choices": [
+ {
+ "delta": {
+ "tool_calls": [
+ {
+ "index": 0,
+ "id": "call_1",
+ "type": "function",
+ "function": {"name": "lookup", "arguments": None},
+ }
+ ]
+ }
+ }
+ ]
+ },
+ )
+ utils.process_chunk(
+ scope,
+ {
+ "choices": [
+ {
+ "delta": {
+ "tool_calls": [
+ {
+ "index": 0,
+ "function": {"arguments": '{"q":"weather"}'},
+ }
+ ]
+ }
+ }
+ ]
+ },
+ )
+
+ assert scope._tools[0]["function"]["name"] == "lookup"
+ assert scope._tools[0]["function"]["arguments"] == '{"q":"weather"}'
+
+
[email protected]("utils", PROVIDERS)
+def test_streaming_parallel_tool_calls_use_index(utils):
+ scope = _scope()
+
+ utils.process_chunk(
+ scope,
+ {
+ "choices": [
+ {
+ "delta": {
+ "tool_calls": [
+ {
+ "index": 0,
+ "id": "call_1",
+ "type": "function",
+ "function": {
+ "name": "get_weather",
+ "arguments": None,
+ },
+ },
+ {
+ "index": 1,
+ "id": "call_2",
+ "type": "function",
+ "function": {"name": "get_time", "arguments": ""},
+ },
+ ]
+ }
+ }
+ ]
+ },
+ )
+ utils.process_chunk(
+ scope,
+ {
+ "choices": [
+ {
+ "delta": {
+ "tool_calls": [
+ {
+ "index": 0,
+ "function": {"arguments": '{"city":"nyc"}'},
+ },
+ {
+ "index": 1,
+ "function": {"arguments": '{"tz":"UTC"}'},
+ },
+ ]
+ }
+ }
+ ]
+ },
+ )
+
+ assert [tool["id"] for tool in scope._tools] == ["call_1", "call_2"]
+ assert [tool["function"]["name"] for tool in scope._tools] == [
+ "get_weather",
+ "get_time",
+ ]
+ assert [tool["function"]["arguments"] for tool in scope._tools] == [
+ '{"city":"nyc"}',
+ '{"tz":"UTC"}',
+ ]