Skip to content

Same agent Smaller bill. The tools and infrastructure to get more work from your AI. Fewer tokens. Less compute. Lower cost.

fix(python-sdk): handle chunked content from Mistral reasoning models (#1598)

HarianthKbed343f5f913
diff --git a/sdk/python/src/openlit/instrumentation/mistral/utils.py b/sdk/python/src/openlit/instrumentation/mistral/utils.py
index ffde6a8a..7d9e80b5 100644
--- a/sdk/python/src/openlit/instrumentation/mistral/utils.py
+++ b/sdk/python/src/openlit/instrumentation/mistral/utils.py
@@ -9,6 +9,7 @@ import time
from opentelemetry.trace import Status, StatusCode
from openlit.__helpers import (
+ append_scope_reasoning,
apply_agent_version_attributes,
build_system_instructions_from_messages,
build_tool_definitions,
@@ -21,6 +22,7 @@ from openlit.__helpers import (
otel_event,
record_completion_metrics,
record_embedding_metrics,
+ set_span_reasoning_content,
truncate_message_content,
)
from openlit.semcov import SemanticConvention
@@ -29,6 +31,31 @@ from openlit._config import OpenlitConfig
logger = logging.getLogger(__name__)
+def split_content_chunks(content):
+ """
+ Split assistant content into (text, reasoning). Reasoning models answer with
+ a list of chunks, a thinking chunk followed by text, instead of a string.
+ """
+
+ if not content:
+ return "", ""
+ if isinstance(content, str):
+ return content, ""
+ text_parts = []
+ reasoning_parts = []
+ for chunk in content:
+ if not isinstance(chunk, dict):
+ continue
+ chunk_type = chunk.get("type")
+ if chunk_type == "thinking":
+ for thought in chunk.get("thinking") or []:
+ if isinstance(thought, dict) and thought.get("text"):
+ reasoning_parts.append(thought["text"])
+ elif chunk_type == "text" and chunk.get("text"):
+ text_parts.append(chunk["text"])
+ return "".join(text_parts), "".join(reasoning_parts)
+
+
def format_content(messages):
"""
Process a list of messages to extract content.
@@ -300,9 +327,10 @@ def process_chunk(scope, chunk):
if choices and "delta" in choices[0]:
delta = choices[0]["delta"]
- content = delta.get("content")
- if content:
- scope._llmresponse += content
+ text, reasoning = split_content_chunks(delta.get("content"))
+ if text:
+ scope._llmresponse += text
+ append_scope_reasoning(scope, reasoning)
# Handle tool calls in streaming - optimized
delta_tools = delta.get("tool_calls")
@@ -530,6 +558,9 @@ def common_chat_logic(
if capture_message_content:
_set_span_messages_as_array(scope._span, input_msgs, output_msgs)
+ set_span_reasoning_content(
+ scope._span, scope, capture_message_content=capture_message_content
+ )
if system_instr:
scope._span.set_attribute(
SemanticConvention.GEN_AI_SYSTEM_INSTRUCTIONS,
@@ -666,9 +697,13 @@ def process_chat_response(
scope._start_time = start_time
scope._end_time = time.time()
scope._span = span
- scope._llmresponse = " ".join(
- (choice.get("message", {}).get("content") or "")
+ contents = [
+ split_content_chunks(choice.get("message", {}).get("content"))
for choice in response_dict.get("choices", [])
+ ]
+ scope._llmresponse = " ".join(text for text, _ in contents)
+ scope._reasoning_content = " ".join(
+ reasoning for _, reasoning in contents if reasoning
)
scope._response_id = response_dict.get("id")
scope._response_model = response_dict.get("model")
diff --git a/sdk/python/tests/test_mistral_chunked_content.py b/sdk/python/tests/test_mistral_chunked_content.py
new file mode 100644
index 00000000..fb17d6e5
--- /dev/null
+++ b/sdk/python/tests/test_mistral_chunked_content.py
@@ -0,0 +1,203 @@
+# pylint: disable=protected-access, missing-function-docstring
+"""Regression tests: Mistral reasoning models answer with a list of content
+chunks (a thinking chunk, then the text) instead of a string. While
+streaming, thinking arrives as a chunk list, the transition event can hold
+a closing thinking chunk and the first text chunk, and the rest of the
+answer arrives as a plain string. The complete and stream wrappers must
+record that answer and the reasoning rather than fail. These tests drive
+the real wrapper factories with responses and chunks built from the
+mistralai SDK's own models, so no API key is needed.
+"""
+
+from mistralai.client.models import (
+ ChatCompletionResponse,
+ CompletionChunk,
+ CompletionEvent,
+ CompletionResponseStreamChoice,
+ DeltaMessage,
+)
+from opentelemetry.sdk.trace import TracerProvider
+from opentelemetry.sdk.trace.export import SimpleSpanProcessor
+from opentelemetry.sdk.trace.export.in_memory_span_exporter import (
+ InMemorySpanExporter,
+)
+from opentelemetry.trace import StatusCode
+
+from openlit._config import OpenlitConfig
+from openlit.instrumentation.mistral import mistral as sync_mod
+from openlit.semcov import SemanticConvention
+
+REASONING = "The user asks for the capital of France. It is Paris."
+ANSWER = "Paris."
+REQUEST_KWARGS = {
+ "model": "magistral-medium-latest",
+ "messages": [{"role": "user", "content": "What is the capital of France?"}],
+}
+
+MAGISTRAL_RESPONSE = ChatCompletionResponse.model_validate(
+ {
+ "id": "b3f1d1f0e8d94f8f9d4b8d7e5a1c2b3d",
+ "object": "chat.completion",
+ "created": 1758000000,
+ "model": "magistral-medium-latest",
+ "choices": [
+ {
+ "index": 0,
+ "message": {
+ "role": "assistant",
+ "content": [
+ {"type": "thinking", "thinking": [{"type": "text", "text": REASONING}]},
+ {"type": "text", "text": ANSWER},
+ ],
+ "tool_calls": None,
+ },
+ "finish_reason": "stop",
+ }
+ ],
+ "usage": {"prompt_tokens": 20, "total_tokens": 61, "completion_tokens": 41},
+ }
+)
+
+
+def _event(content, finish_reason=None, usage=None):
+ return CompletionEvent(
+ data=CompletionChunk(
+ id="b3f1d1f0e8d94f8f9d4b8d7e5a1c2b3d",
+ model="magistral-medium-latest",
+ choices=[
+ CompletionResponseStreamChoice(
+ index=0,
+ delta=DeltaMessage(role="assistant", content=content),
+ finish_reason=finish_reason,
+ )
+ ],
+ usage=usage,
+ )
+ )
+
+
+def _magistral_stream():
+ """The shape Mistral documents for reasoning_effort="high".
+
+ Thinking is a chunk list. The transition event closes that chunk and
+ carries the first text fragment. Later answer deltas are plain strings.
+ """
+
+ return iter(
+ [
+ _event(
+ [
+ {
+ "type": "thinking",
+ "thinking": [{"type": "text", "text": REASONING}],
+ }
+ ]
+ ),
+ _event(
+ [
+ {"type": "thinking", "thinking": [], "closed": True},
+ {"type": "text", "text": "Par"},
+ ]
+ ),
+ _event("is."),
+ _event(
+ None,
+ finish_reason="stop",
+ usage={
+ "prompt_tokens": 20,
+ "completion_tokens": 41,
+ "total_tokens": 61,
+ },
+ ),
+ ]
+ )
+
+
+def _tracer_with_exporter():
+ OpenlitConfig.reset_to_defaults()
+ exporter = InMemorySpanExporter()
+ provider = TracerProvider()
+ provider.add_span_processor(SimpleSpanProcessor(exporter))
+ return provider.get_tracer(__name__), exporter
+
+
+def _factory(make, tracer, capture_message_content=True):
+ return make(
+ version="test",
+ environment="test",
+ application_name="test",
+ tracer=tracer,
+ pricing_info={},
+ capture_message_content=capture_message_content,
+ metrics=None,
+ disable_metrics=True,
+ event_provider=None,
+ )
+
+
+def _completion_text(span):
+ return span.attributes.get(SemanticConvention.GEN_AI_OUTPUT_MESSAGES)
+
+
+def test_complete_records_chunked_answer():
+ tracer, exporter = _tracer_with_exporter()
+ wrapper = _factory(sync_mod.complete, tracer)
+
+ wrapper(lambda *a, **k: MAGISTRAL_RESPONSE, None, (), REQUEST_KWARGS)
+
+ spans = exporter.get_finished_spans()
+ assert len(spans) == 1
+ span = spans[0]
+ assert span.status.status_code != StatusCode.ERROR, "the span was marked as failed"
+ completion = _completion_text(span)
+ assert completion is not None, "the answer was not recorded"
+ assert ANSWER in completion
+ assert REASONING not in completion, "reasoning is recorded separately, not as the answer"
+ assert span.attributes.get(SemanticConvention.GEN_AI_CONTENT_REASONING) == REASONING
+
+
+def test_stream_records_chunked_answer_and_keeps_yielding():
+ tracer, exporter = _tracer_with_exporter()
+ wrapper = _factory(sync_mod.stream, tracer)
+
+ stream = wrapper(lambda *a, **k: _magistral_stream(), None, (), REQUEST_KWARGS)
+ events = list(stream)
+
+ assert len(events) == 4, "every chunk must still reach the caller"
+ spans = exporter.get_finished_spans()
+ assert len(spans) == 1
+ span = spans[0]
+ assert span.status.status_code != StatusCode.ERROR
+ completion = _completion_text(span)
+ assert completion is not None, "the streamed answer was not recorded"
+ assert ANSWER in completion
+ assert REASONING not in completion
+ assert span.attributes.get(SemanticConvention.GEN_AI_CONTENT_REASONING) == REASONING
+
+
+def test_capture_disabled_omits_reasoning_and_answer():
+ tracer, exporter = _tracer_with_exporter()
+ wrapper = _factory(sync_mod.complete, tracer, capture_message_content=False)
+
+ returned = wrapper(lambda *a, **k: MAGISTRAL_RESPONSE, None, (), REQUEST_KWARGS)
+
+ assert returned is MAGISTRAL_RESPONSE
+ spans = exporter.get_finished_spans()
+ assert len(spans) == 1
+ span = spans[0]
+ assert span.status.status_code != StatusCode.ERROR
+ assert SemanticConvention.GEN_AI_CONTENT_REASONING not in span.attributes
+ assert SemanticConvention.GEN_AI_OUTPUT_MESSAGES not in span.attributes
+
+
+def test_reasoning_respects_max_content_length():
+ tracer, exporter = _tracer_with_exporter()
+ OpenlitConfig.max_content_length = 8
+ wrapper = _factory(sync_mod.complete, tracer)
+
+ wrapper(lambda *a, **k: MAGISTRAL_RESPONSE, None, (), REQUEST_KWARGS)
+
+ spans = exporter.get_finished_spans()
+ assert len(spans) == 1
+ reasoning = spans[0].attributes.get(SemanticConvention.GEN_AI_CONTENT_REASONING)
+ assert reasoning == REASONING[:8] + "..."