Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix: normalize reasoning stream markers (#19216)

buua436ef8b424888ff
diff --git a/api/db/services/dialog_service.py b/api/db/services/dialog_service.py
index d6c607568e..6ce31c5f8b 100644
--- a/api/db/services/dialog_service.py
+++ b/api/db/services/dialog_service.py
@@ -1598,12 +1598,8 @@ def tts(tts_mdl, text):
class _ThinkStreamState:
def __init__(self) -> None:
self.full_text = ""
- self.last_idx = 0
self.last_model_full = ""
self.in_think = False
- self.close_pending = False
- self.pending_after_close = ""
- self.think_buffer = ""
self.answer_buffer = ""
@@ -1635,13 +1631,6 @@ async def _stream_with_think_delta(stream_iter, min_tokens: int = 16):
return out
return None
- def _flush_think_buffer():
- if not state.think_buffer:
- return None
- out = state.think_buffer
- state.think_buffer = ""
- return out
-
def _flush_answer_buffer():
if not state.answer_buffer:
return None
@@ -1663,26 +1652,6 @@ async def _stream_with_think_delta(stream_iter, min_tokens: int = 16):
state.full_text += new_part
pending = new_part
- if state.close_pending and "</think>" not in pending:
- state.close_pending = False
- think_piece = _flush_think_buffer()
- if think_piece is not None:
- yield ("text", think_piece, state)
- state.in_think = False
- yield ("marker", "</think>", state)
- if state.pending_after_close:
- answer_piece = state.pending_after_close
- state.pending_after_close = ""
- out = _emit_text("answer", answer_piece)
- if out is not None:
- yield ("text", out, state)
- answer_piece = re.sub(r"</?think>", "", pending or "")
- if answer_piece:
- out = _emit_text("answer", answer_piece)
- if out is not None:
- yield ("text", out, state)
- continue
-
while pending:
open_idx = pending.find("<think>")
close_idx = pending.find("</think>")
@@ -1709,9 +1678,6 @@ async def _stream_with_think_delta(stream_iter, min_tokens: int = 16):
answer_piece = _flush_answer_buffer()
if answer_piece is not None:
yield ("text", answer_piece, state)
- think_piece = _flush_think_buffer()
- if think_piece is not None:
- yield ("text", think_piece, state)
state.in_think = True
yield ("marker", "<think>", state)
continue
@@ -1725,32 +1691,15 @@ async def _stream_with_think_delta(stream_iter, min_tokens: int = 16):
if out is not None:
yield ("text", out, state)
after_visible = re.sub(r"</?think>", "", after or "")
- if after_visible.strip():
- think_piece = _flush_think_buffer()
- if think_piece is not None:
- yield ("text", think_piece, state)
+ if state.in_think:
state.in_think = False
yield ("marker", "</think>", state)
- pending = after_visible
- continue
- state.close_pending = True
- if after_visible:
- state.pending_after_close += after_visible
- pending = ""
- break
+ pending = after_visible
+ continue
- if state.think_buffer:
- yield ("text", state.think_buffer, state)
- state.think_buffer = ""
- if state.close_pending:
- state.in_think = False
- yield ("marker", "</think>", state)
if state.answer_buffer:
yield ("text", state.answer_buffer, state)
state.answer_buffer = ""
- if state.pending_after_close:
- yield ("text", state.pending_after_close, state)
- state.pending_after_close = ""
async def async_ask(question, kb_ids, tenant_id, chat_llm_name=None, search_config={}, search_id=None):
diff --git a/rag/llm/chat_model.py b/rag/llm/chat_model.py
index 2e9d9ef618..9018544611 100644
--- a/rag/llm/chat_model.py
+++ b/rag/llm/chat_model.py
@@ -69,6 +69,7 @@ ERROR_PREFIX = "**ERROR**"
LENGTH_NOTIFICATION_CN = "······\n由于大模型的上下文窗口大小限制,回答已经被大模型截断。"
LENGTH_NOTIFICATION_EN = "...\nThe answer is truncated by your chosen LLM due to its limitation on context length."
+
# Generation parameters that are safe to forward to the underlying completion
# call. `gen_conf` originates from a chat assistant's `llm_setting`, which can
# also carry RAGFlow-internal metadata (e.g. `model_type`). Anything outside
@@ -304,6 +305,8 @@ class Base(ABC):
async def _async_chat_streamly(self, history, gen_conf, **kwargs):
logging.info("[HISTORY STREAMLY]" + json.dumps(history, ensure_ascii=False, indent=4))
reasoning_start = False
+ answer = ""
+ generated_text = ""
gen_conf, extra_request_kwargs = _apply_model_family_policies(
self.model_name,
@@ -325,25 +328,46 @@ class Base(ABC):
resp.choices[0].delta.content = ""
_reasoning = getattr(resp.choices[0].delta, "reasoning_content", None) or getattr(resp.choices[0].delta, "reasoning", None)
if kwargs.get("with_reasoning", True) and _reasoning:
- ans = ""
if not reasoning_start:
reasoning_start = True
- ans = "<think>"
- ans += _reasoning + "</think>"
+ yield "<think>", 0
+ tol = total_token_count_from_response(resp)
+ if not tol:
+ tol = num_tokens_from_string(resp.choices[0].delta.content)
+ generated_text += _reasoning
+ yield _reasoning, tol
+ if resp.choices[0].delta.content:
+ reasoning_start = False
+ yield "</think>", 0
+ answer += resp.choices[0].delta.content
+ generated_text += resp.choices[0].delta.content
+ yield resp.choices[0].delta.content, 0
+ if getattr(resp.choices[0], "finish_reason", "") == "length":
+ if reasoning_start:
+ reasoning_start = False
+ yield "</think>", 0
+ yield LENGTH_NOTIFICATION_CN if is_chinese(answer or generated_text) else LENGTH_NOTIFICATION_EN, 0
+ continue
else:
- reasoning_start = False
+ if reasoning_start and resp.choices[0].delta.content:
+ reasoning_start = False
+ yield "</think>", 0
ans = resp.choices[0].delta.content
+ answer += ans
tol = total_token_count_from_response(resp)
if not tol:
tol = num_tokens_from_string(resp.choices[0].delta.content)
finish_reason = resp.choices[0].finish_reason if hasattr(resp.choices[0], "finish_reason") else ""
- if finish_reason == "length":
- if is_chinese(ans):
- ans += LENGTH_NOTIFICATION_CN
- else:
- ans += LENGTH_NOTIFICATION_EN
yield ans, tol
+ if finish_reason == "length":
+ if reasoning_start:
+ reasoning_start = False
+ yield "</think>", 0
+ yield LENGTH_NOTIFICATION_CN if is_chinese(answer) else LENGTH_NOTIFICATION_EN, 0
+
+ if reasoning_start:
+ yield "</think>", 0
async def async_chat_streamly(self, system, history, gen_conf: dict | None = None, **kwargs):
gen_conf = dict(gen_conf or {})
@@ -670,6 +694,7 @@ class Base(ABC):
final_tool_calls = {}
answer = ""
+ generated_text = ""
round_estimate = 0
round_usage = None
@@ -699,14 +724,20 @@ class Base(ABC):
_reasoning = getattr(delta, "reasoning_content", None) or getattr(delta, "reasoning", None)
if _reasoning:
- ans = ""
+ generated_text += _reasoning
if not reasoning_start:
reasoning_start = True
- ans = "<think>"
- ans += _reasoning + "</think>"
- yield ans
+ yield "<think>"
+ yield _reasoning
+ if delta.content:
+ reasoning_start = False
+ yield "</think>"
+ answer += delta.content
+ yield delta.content
else:
- reasoning_start = False
+ if reasoning_start and delta.content:
+ reasoning_start = False
+ yield "</think>"
answer += delta.content
yield delta.content
@@ -715,7 +746,13 @@ class Base(ABC):
finish_reason = getattr(resp.choices[0], "finish_reason", "")
if finish_reason == "length":
- yield self._length_stop("")
+ if reasoning_start:
+ reasoning_start = False
+ yield "</think>"
+ yield LENGTH_NOTIFICATION_CN if is_chinese(answer or generated_text) else LENGTH_NOTIFICATION_EN
+
+ if reasoning_start:
+ yield "</think>"
# Commit this round's tokens (each round is a separate provider
# request — accumulate, never overwrite).
@@ -748,7 +785,9 @@ class Base(ABC):
args = json_repair.loads(tc.function.arguments)
except Exception:
args = {}
- yield f"<think>Running the {tc.function.name} tool...</think>"
+ yield "<think>"
+ yield f"Running the {tc.function.name} tool..."
+ yield "</think>"
results = await asyncio.gather(*[_exec_tool(tc) for tc in tcs])
# Terminal-tool short-circuit: stream a terminal tool's
@@ -2049,6 +2088,8 @@ class LiteLLMBase(ABC):
gen_conf = self._clean_conf(gen_conf)
reasoning_start = False
total_tokens = 0
+ answer = ""
+ generated_text = ""
# Reset so a stale split from a previous call can't leak into this one.
self.last_usage = {"prompt_tokens": 0, "completion_tokens": 0, "total_tokens": 0}
@@ -2086,27 +2127,43 @@ class LiteLLMBase(ABC):
_reasoning = getattr(delta, "reasoning_content", None) or getattr(delta, "reasoning", None)
if kwargs.get("with_reasoning", True) and _reasoning:
- ans = ""
if not reasoning_start:
reasoning_start = True
- ans = "<think>"
- ans += _reasoning + "</think>"
+ yield "<think>"
+ yield _reasoning
+ generated_text += _reasoning
+ if delta.content:
+ reasoning_start = False
+ yield "</think>"
+ yield delta.content
+ answer += delta.content
+ generated_text += delta.content
+ if getattr(resp.choices[0], "finish_reason", "") == "length":
+ if reasoning_start:
+ reasoning_start = False
+ yield "</think>"
+ yield LENGTH_NOTIFICATION_CN if is_chinese(answer or generated_text) else LENGTH_NOTIFICATION_EN
+ continue
else:
- reasoning_start = False
+ if reasoning_start and delta.content:
+ reasoning_start = False
+ yield "</think>"
ans = delta.content
+ answer += ans
if not _usage["total_tokens"]:
# No authoritative usage yet: keep a running estimate as fallback.
total_tokens += num_tokens_from_string(delta.content)
finish_reason = resp.choices[0].finish_reason if hasattr(resp.choices[0], "finish_reason") else ""
- if finish_reason == "length":
- if is_chinese(ans):
- ans += LENGTH_NOTIFICATION_CN
- else:
- ans += LENGTH_NOTIFICATION_EN
-
yield ans
+ if finish_reason == "length":
+ if reasoning_start:
+ reasoning_start = False
+ yield "</think>"
+ yield LENGTH_NOTIFICATION_CN if is_chinese(answer) else LENGTH_NOTIFICATION_EN
+ if reasoning_start:
+ yield "</think>"
yield total_tokens
return
except Exception as e:
@@ -2391,6 +2448,7 @@ class LiteLLMBase(ABC):
final_tool_calls = {}
answer = ""
+ generated_text = ""
round_usage = None
round_estimate = 0
# Per-round filter for providers (MiniMax) whose control tokens
@@ -2424,16 +2482,28 @@ class LiteLLMBase(ABC):
_reasoning = getattr(delta, "reasoning_content", None) or getattr(delta, "reasoning", None)
if _reasoning:
+ generated_text += _reasoning
if self._need_reasoning_content_back():
reasoning_content += _reasoning
- ans = ""
if not reasoning_start:
reasoning_start = True
- ans = "<think>"
- ans += _reasoning + "</think>"
- yield ans
+ yield "<think>"
+ yield _reasoning
+ if delta.content:
+ reasoning_start = False
+ yield "</think>"
+ answer += delta.content
+ generated_text += delta.content
+ if _sanitizer is not None:
+ emitted = _sanitizer.feed(delta.content)
+ if emitted:
+ yield emitted
+ else:
+ yield delta.content
else:
- reasoning_start = False
+ if reasoning_start and delta.content:
+ reasoning_start = False
+ yield "</think>"
answer += delta.content
if _sanitizer is not None:
emitted = _sanitizer.feed(delta.content)
@@ -2447,7 +2517,13 @@ class LiteLLMBase(ABC):
finish_reason = getattr(resp.choices[0], "finish_reason", "")
if finish_reason == "length":
- yield self._length_stop("")
+ if reasoning_start:
+ reasoning_start = False
+ yield "</think>"
+ yield LENGTH_NOTIFICATION_CN if is_chinese(answer or generated_text) else LENGTH_NOTIFICATION_EN
+
+ if reasoning_start:
+ yield "</think>"
# Flush any held-back (sanitized) answer content for this round.
if _sanitizer is not None:
@@ -2485,7 +2561,9 @@ class LiteLLMBase(ABC):
args = json_repair.loads(tc.function.arguments)
except Exception:
args = {}
- yield f"<think>Running the {tc.function.name} tool...</think>"
+ yield "<think>"
+ yield f"Running the {tc.function.name} tool..."
+ yield "</think>"
results = await asyncio.gather(*[_exec_tool(tc) for tc in tcs])
# Terminal-tool short-circuit: a terminal tool already
diff --git a/test/unit_test/common/test_think_stream_parser.py b/test/unit_test/common/test_think_stream_parser.py
index ec9ac654d0..25e1b46134 100644
--- a/test/unit_test/common/test_think_stream_parser.py
+++ b/test/unit_test/common/test_think_stream_parser.py
@@ -49,40 +49,11 @@ CASES = [
{
"min_tokens": 16,
"chunks": [
- "<think>We</think>",
- " need</think>",
- " to</think>",
- " respond</think>",
- " to</think>",
- " the</think>",
- " user</think>",
- "'s</think>",
- " greeting</think>",
- ' "</think>',
- "hello</think>",
- '".</think>',
- " The</think>",
- " assistant</think>",
- " should</think>",
- " be</think>",
- " friendly</think>",
- " and</think>",
- " helpful</think>",
- ".</think>",
- " A</think>",
- " simple</think>",
- " greeting</think>",
- " back</think>",
- " is</think>",
- " appropriate</think>",
- ",</think>",
- " perhaps</think>",
- " with</think>",
- " an</think>",
- " offer</think>",
- " of</think>",
- " assistance</think>",
- ".</think>",
+ "<think>",
+ "We need to respond to the user's greeting ",
+ '"hello". The assistant should be friendly and helpful. ',
+ "A simple greeting back is appropriate, perhaps with an offer of assistance.",
+ "</think>",
"Hello",
"!",
" How",
@@ -96,6 +67,7 @@ CASES = [
"expected": {
"think": 'We need to respond to the user\'s greeting "hello". The assistant should be friendly and helpful. A simple greeting back is appropriate, perhaps with an offer of assistance.',
"answer": "Hello! How can I assist you today?",
+ "markers": ["<think>", "</think>"],
},
},
),
@@ -104,60 +76,13 @@ CASES = [
{
"min_tokens": 16,
"chunks": [
- "<think>We</think>",
- " need</think>",
- " to</think>",
- " respond</think>",
- " to</think>",
- " the</think>",
- " user</think>",
- "'s</think>",
- ' "</think>',
- "hello</think>",
- '"</think>',
- " again</think>",
- ".</think>",
- " The</think>",
- " user</think>",
- " just</think>",
- " said</think>",
- ' "</think>',
- "hello</think>",
- '"</think>',
- " after</think>",
- " I</think>",
- " already</think>",
- " responded</think>",
- ".</think>",
- " Possibly</think>",
- " they</think>",
- "'re</think>",
- " testing</think>",
- " or</think>",
- " just</think>",
- " greeting</think>",
- " again</think>",
- ".</think>",
- " I</think>",
- "'ll</think>",
- " respond</think>",
- " in</think>",
- " a</think>",
- " friendly</think>",
- " manner</think>",
- ",</think>",
- " perhaps</think>",
- " acknowledging</think>",
- " the</think>",
- " repeated</think>",
- " greeting</think>",
- " and</think>",
- " inviting</think>",
- " them</think>",
- " to</think>",
- " ask</think>",
- " something</think>",
- ".</think>",
+ "<think>",
+ 'We need to respond to the user\'s "hello" again. ',
+ 'The user just said "hello" after I already responded. ',
+ "Possibly they're testing or just greeting again. ",
+ "I'll respond in a friendly manner, perhaps acknowledging the repeated greeting ",
+ "and inviting them to ask something.",
+ "</think>",
"Hello",
" again",
"!",
@@ -172,6 +97,7 @@ CASES = [
"expected": {
"think": 'We need to respond to the user\'s "hello" again. The user just said "hello" after I already responded. Possibly they\'re testing or just greeting again. I\'ll respond in a friendly manner, perhaps acknowledging the repeated greeting and inviting them to ask something.',
"answer": "Hello again! How can I help you today?",
+ "markers": ["<think>", "</think>"],
},
},
),
@@ -266,7 +192,7 @@ CASES = [
"expected": {
"think": "重复开始",
"answer": "答案尾巴",
- "markers": ["<think>", "</think>", "</think>"],
+ "markers": ["<think>", "</think>"],
},
},
),
diff --git a/test/unit_test/rag/llm/test_chat_model_reasoning_stream.py b/test/unit_test/rag/llm/test_chat_model_reasoning_stream.py
new file mode 100644
index 0000000000..f9b56b872f
--- /dev/null
+++ b/test/unit_test/rag/llm/test_chat_model_reasoning_stream.py
@@ -0,0 +1,154 @@
+#
+# Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+from types import SimpleNamespace
+
+import pytest
+
+from rag.llm import SupportedLiteLLMProvider, chat_model
+from rag.llm.chat_model import LENGTH_NOTIFICATION_CN, LENGTH_NOTIFICATION_EN, LiteLLMBase
+
+pytestmark = pytest.mark.p1
+
+
+def _stream_chunk(*, content="", reasoning=None, finish_reason=None):
+ return SimpleNamespace(
+ choices=[
+ SimpleNamespace(
+ delta=SimpleNamespace(content=content, reasoning_content=reasoning, reasoning=None, tool_calls=None),
+ finish_reason=finish_reason,
+ )
+ ],
+ usage=None,
+ )
+
+
+def _stream(chunks):
+ async def _iterate():
+ for chunk in chunks:
+ yield chunk
+
+ return _iterate()
+
+
+def _make_model():
+ model = LiteLLMBase.__new__(LiteLLMBase)
+ model.model_name = "test-model"
+ model.provider = SupportedLiteLLMProvider.OpenAI
+ model.api_key = "test-key"
+ model.base_url = "https://example.test/v1"
+ model.max_retries = 0
+ model.max_rounds = 0
+ model.timeout = 1
+ model.tools = [
+ {
+ "type": "function",
+ "function": {
+ "name": "lookup",
+ "parameters": {"type": "object", "properties": {}},
+ },
+ }
+ ]
+ model.last_usage = {"prompt_tokens": 0, "completion_tokens": 0, "total_tokens": 0}
+ return model
+
+
+async def _collect(monkeypatch, model, *, chunks, with_tools=False, with_reasoning=True):
+ async def fake_acompletion(**_kwargs):
+ return _stream(chunks)
+
+ monkeypatch.setattr(chat_model.litellm, "acompletion", fake_acompletion)
+ history = [{"role": "user", "content": "hello"}]
+ if with_tools:
+ return [event async for event in model.async_chat_streamly_with_tools("", history, {})]
+ return [event async for event in model.async_chat_streamly("", history, {}, with_reasoning=with_reasoning)]
+
+
[email protected]("with_tools", [False, True])
+async def test_reasoning_stream_emits_one_marker_pair(monkeypatch, with_tools):
+ events = await _collect(
+ monkeypatch,
+ _make_model(),
+ with_tools=with_tools,
+ chunks=[
+ _stream_chunk(reasoning="reasoning one"),
+ _stream_chunk(),
+ _stream_chunk(reasoning=" and two"),
+ _stream_chunk(content="answer", finish_reason="stop"),
+ ],
+ )
+
+ text_events = [event for event in events if isinstance(event, str) and event]
+ assert text_events == ["<think>", "reasoning one", " and two", "</think>", "answer"]
+ assert isinstance(events[-1], int)
+
+
[email protected]("with_tools", [False, True])
+async def test_reasoning_and_answer_in_same_delta_are_both_emitted(monkeypatch, with_tools):
+ events = await _collect(
+ monkeypatch,
+ _make_model(),
+ with_tools=with_tools,
+ chunks=[_stream_chunk(reasoning="reasoning", content="answer", finish_reason="stop")],
+ )
+
+ text_events = [event for event in events if isinstance(event, str) and event]
+ assert text_events == ["<think>", "reasoning", "</think>", "answer"]
+
+
+async def test_reasoning_stream_can_hide_reasoning(monkeypatch):
+ events = await _collect(
+ monkeypatch,
+ _make_model(),
+ with_reasoning=False,
+ chunks=[
+ _stream_chunk(reasoning="hidden"),
+ _stream_chunk(content="answer", finish_reason="stop"),
+ ],
+ )
+
+ assert [event for event in events if isinstance(event, str) and event] == ["answer"]
+ assert isinstance(events[-1], int)
+
+
[email protected]("finish_reason", [None, "length"])
+async def test_reasoning_only_stream_is_closed(monkeypatch, finish_reason):
+ events = await _collect(monkeypatch, _make_model(), chunks=[_stream_chunk(reasoning="思考", finish_reason=finish_reason)])
+
+ assert events[:3] == ["<think>", "思考", "</think>"]
+ assert events.count("<think>") == 1
+ assert events.count("</think>") == 1
+ if finish_reason == "length":
+ assert LENGTH_NOTIFICATION_CN in events
+ assert isinstance(events[-1], int)
+
+
[email protected]("with_tools", [False, True])
+async def test_length_notification_uses_answer_language(monkeypatch, with_tools):
+ events = await _collect(
+ monkeypatch,
+ _make_model(),
+ with_tools=with_tools,
+ chunks=[_stream_chunk(content="这是答案", finish_reason="length")],
+ )
+
+ assert LENGTH_NOTIFICATION_CN in events
+ assert LENGTH_NOTIFICATION_EN not in events