RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix: normalize reasoning stream markers (#19216)
buua436ef8b424888ff
diff --git a/api/db/services/dialog_service.py b/api/db/services/dialog_service.pyindex d6c607568e..6ce31c5f8b 100644--- a/api/db/services/dialog_service.py+++ b/api/db/services/dialog_service.py@@ -1598,12 +1598,8 @@ def tts(tts_mdl, text):class _ThinkStreamState:def __init__(self) -> None:self.full_text = ""- self.last_idx = 0self.last_model_full = ""self.in_think = False- self.close_pending = False- self.pending_after_close = ""- self.think_buffer = ""self.answer_buffer = ""@@ -1635,13 +1631,6 @@ async def _stream_with_think_delta(stream_iter, min_tokens: int = 16):return outreturn None- def _flush_think_buffer():- if not state.think_buffer:- return None- out = state.think_buffer- state.think_buffer = ""- return out-def _flush_answer_buffer():if not state.answer_buffer:return None@@ -1663,26 +1652,6 @@ async def _stream_with_think_delta(stream_iter, min_tokens: int = 16):state.full_text += new_partpending = new_part- if state.close_pending and "</think>" not in pending:- state.close_pending = False- think_piece = _flush_think_buffer()- if think_piece is not None:- yield ("text", think_piece, state)- state.in_think = False- yield ("marker", "</think>", state)- if state.pending_after_close:- answer_piece = state.pending_after_close- state.pending_after_close = ""- out = _emit_text("answer", answer_piece)- if out is not None:- yield ("text", out, state)- answer_piece = re.sub(r"</?think>", "", pending or "")- if answer_piece:- out = _emit_text("answer", answer_piece)- if out is not None:- yield ("text", out, state)- continue-while pending:open_idx = pending.find("<think>")close_idx = pending.find("</think>")@@ -1709,9 +1678,6 @@ async def _stream_with_think_delta(stream_iter, min_tokens: int = 16):answer_piece = _flush_answer_buffer()if answer_piece is not None:yield ("text", answer_piece, state)- think_piece = _flush_think_buffer()- if think_piece is not None:- yield ("text", think_piece, state)state.in_think = Trueyield ("marker", "<think>", state)continue@@ -1725,32 +1691,15 @@ async def _stream_with_think_delta(stream_iter, min_tokens: int = 16):if out is not None:yield ("text", out, state)after_visible = re.sub(r"</?think>", "", after or "")- if after_visible.strip():- think_piece = _flush_think_buffer()- if think_piece is not None:- yield ("text", think_piece, state)+ if state.in_think:state.in_think = Falseyield ("marker", "</think>", state)- pending = after_visible- continue- state.close_pending = True- if after_visible:- state.pending_after_close += after_visible- pending = ""- break+ pending = after_visible+ continue- if state.think_buffer:- yield ("text", state.think_buffer, state)- state.think_buffer = ""- if state.close_pending:- state.in_think = False- yield ("marker", "</think>", state)if state.answer_buffer:yield ("text", state.answer_buffer, state)state.answer_buffer = ""- if state.pending_after_close:- yield ("text", state.pending_after_close, state)- state.pending_after_close = ""async def async_ask(question, kb_ids, tenant_id, chat_llm_name=None, search_config={}, search_id=None):diff --git a/rag/llm/chat_model.py b/rag/llm/chat_model.pyindex 2e9d9ef618..9018544611 100644--- a/rag/llm/chat_model.py+++ b/rag/llm/chat_model.py@@ -69,6 +69,7 @@ ERROR_PREFIX = "**ERROR**"LENGTH_NOTIFICATION_CN = "······\n由于大模型的上下文窗口大小限制,回答已经被大模型截断。"LENGTH_NOTIFICATION_EN = "...\nThe answer is truncated by your chosen LLM due to its limitation on context length."+# Generation parameters that are safe to forward to the underlying completion# call. `gen_conf` originates from a chat assistant's `llm_setting`, which can# also carry RAGFlow-internal metadata (e.g. `model_type`). Anything outside@@ -304,6 +305,8 @@ class Base(ABC):async def _async_chat_streamly(self, history, gen_conf, **kwargs):logging.info("[HISTORY STREAMLY]" + json.dumps(history, ensure_ascii=False, indent=4))reasoning_start = False+ answer = ""+ generated_text = ""gen_conf, extra_request_kwargs = _apply_model_family_policies(self.model_name,@@ -325,25 +328,46 @@ class Base(ABC):resp.choices[0].delta.content = ""_reasoning = getattr(resp.choices[0].delta, "reasoning_content", None) or getattr(resp.choices[0].delta, "reasoning", None)if kwargs.get("with_reasoning", True) and _reasoning:- ans = ""if not reasoning_start:reasoning_start = True- ans = "<think>"- ans += _reasoning + "</think>"+ yield "<think>", 0+ tol = total_token_count_from_response(resp)+ if not tol:+ tol = num_tokens_from_string(resp.choices[0].delta.content)+ generated_text += _reasoning+ yield _reasoning, tol+ if resp.choices[0].delta.content:+ reasoning_start = False+ yield "</think>", 0+ answer += resp.choices[0].delta.content+ generated_text += resp.choices[0].delta.content+ yield resp.choices[0].delta.content, 0+ if getattr(resp.choices[0], "finish_reason", "") == "length":+ if reasoning_start:+ reasoning_start = False+ yield "</think>", 0+ yield LENGTH_NOTIFICATION_CN if is_chinese(answer or generated_text) else LENGTH_NOTIFICATION_EN, 0+ continueelse:- reasoning_start = False+ if reasoning_start and resp.choices[0].delta.content:+ reasoning_start = False+ yield "</think>", 0ans = resp.choices[0].delta.content+ answer += anstol = total_token_count_from_response(resp)if not tol:tol = num_tokens_from_string(resp.choices[0].delta.content)finish_reason = resp.choices[0].finish_reason if hasattr(resp.choices[0], "finish_reason") else ""- if finish_reason == "length":- if is_chinese(ans):- ans += LENGTH_NOTIFICATION_CN- else:- ans += LENGTH_NOTIFICATION_ENyield ans, tol+ if finish_reason == "length":+ if reasoning_start:+ reasoning_start = False+ yield "</think>", 0+ yield LENGTH_NOTIFICATION_CN if is_chinese(answer) else LENGTH_NOTIFICATION_EN, 0++ if reasoning_start:+ yield "</think>", 0async def async_chat_streamly(self, system, history, gen_conf: dict | None = None, **kwargs):gen_conf = dict(gen_conf or {})@@ -670,6 +694,7 @@ class Base(ABC):final_tool_calls = {}answer = ""+ generated_text = ""round_estimate = 0round_usage = None@@ -699,14 +724,20 @@ class Base(ABC):_reasoning = getattr(delta, "reasoning_content", None) or getattr(delta, "reasoning", None)if _reasoning:- ans = ""+ generated_text += _reasoningif not reasoning_start:reasoning_start = True- ans = "<think>"- ans += _reasoning + "</think>"- yield ans+ yield "<think>"+ yield _reasoning+ if delta.content:+ reasoning_start = False+ yield "</think>"+ answer += delta.content+ yield delta.contentelse:- reasoning_start = False+ if reasoning_start and delta.content:+ reasoning_start = False+ yield "</think>"answer += delta.contentyield delta.content@@ -715,7 +746,13 @@ class Base(ABC):finish_reason = getattr(resp.choices[0], "finish_reason", "")if finish_reason == "length":- yield self._length_stop("")+ if reasoning_start:+ reasoning_start = False+ yield "</think>"+ yield LENGTH_NOTIFICATION_CN if is_chinese(answer or generated_text) else LENGTH_NOTIFICATION_EN++ if reasoning_start:+ yield "</think>"# Commit this round's tokens (each round is a separate provider# request — accumulate, never overwrite).@@ -748,7 +785,9 @@ class Base(ABC):args = json_repair.loads(tc.function.arguments)except Exception:args = {}- yield f"<think>Running the {tc.function.name} tool...</think>"+ yield "<think>"+ yield f"Running the {tc.function.name} tool..."+ yield "</think>"results = await asyncio.gather(*[_exec_tool(tc) for tc in tcs])# Terminal-tool short-circuit: stream a terminal tool's@@ -2049,6 +2088,8 @@ class LiteLLMBase(ABC):gen_conf = self._clean_conf(gen_conf)reasoning_start = Falsetotal_tokens = 0+ answer = ""+ generated_text = ""# Reset so a stale split from a previous call can't leak into this one.self.last_usage = {"prompt_tokens": 0, "completion_tokens": 0, "total_tokens": 0}@@ -2086,27 +2127,43 @@ class LiteLLMBase(ABC):_reasoning = getattr(delta, "reasoning_content", None) or getattr(delta, "reasoning", None)if kwargs.get("with_reasoning", True) and _reasoning:- ans = ""if not reasoning_start:reasoning_start = True- ans = "<think>"- ans += _reasoning + "</think>"+ yield "<think>"+ yield _reasoning+ generated_text += _reasoning+ if delta.content:+ reasoning_start = False+ yield "</think>"+ yield delta.content+ answer += delta.content+ generated_text += delta.content+ if getattr(resp.choices[0], "finish_reason", "") == "length":+ if reasoning_start:+ reasoning_start = False+ yield "</think>"+ yield LENGTH_NOTIFICATION_CN if is_chinese(answer or generated_text) else LENGTH_NOTIFICATION_EN+ continueelse:- reasoning_start = False+ if reasoning_start and delta.content:+ reasoning_start = False+ yield "</think>"ans = delta.content+ answer += ansif not _usage["total_tokens"]:# No authoritative usage yet: keep a running estimate as fallback.total_tokens += num_tokens_from_string(delta.content)finish_reason = resp.choices[0].finish_reason if hasattr(resp.choices[0], "finish_reason") else ""- if finish_reason == "length":- if is_chinese(ans):- ans += LENGTH_NOTIFICATION_CN- else:- ans += LENGTH_NOTIFICATION_EN-yield ans+ if finish_reason == "length":+ if reasoning_start:+ reasoning_start = False+ yield "</think>"+ yield LENGTH_NOTIFICATION_CN if is_chinese(answer) else LENGTH_NOTIFICATION_EN+ if reasoning_start:+ yield "</think>"yield total_tokensreturnexcept Exception as e:@@ -2391,6 +2448,7 @@ class LiteLLMBase(ABC):final_tool_calls = {}answer = ""+ generated_text = ""round_usage = Noneround_estimate = 0# Per-round filter for providers (MiniMax) whose control tokens@@ -2424,16 +2482,28 @@ class LiteLLMBase(ABC):_reasoning = getattr(delta, "reasoning_content", None) or getattr(delta, "reasoning", None)if _reasoning:+ generated_text += _reasoningif self._need_reasoning_content_back():reasoning_content += _reasoning- ans = ""if not reasoning_start:reasoning_start = True- ans = "<think>"- ans += _reasoning + "</think>"- yield ans+ yield "<think>"+ yield _reasoning+ if delta.content:+ reasoning_start = False+ yield "</think>"+ answer += delta.content+ generated_text += delta.content+ if _sanitizer is not None:+ emitted = _sanitizer.feed(delta.content)+ if emitted:+ yield emitted+ else:+ yield delta.contentelse:- reasoning_start = False+ if reasoning_start and delta.content:+ reasoning_start = False+ yield "</think>"answer += delta.contentif _sanitizer is not None:emitted = _sanitizer.feed(delta.content)@@ -2447,7 +2517,13 @@ class LiteLLMBase(ABC):finish_reason = getattr(resp.choices[0], "finish_reason", "")if finish_reason == "length":- yield self._length_stop("")+ if reasoning_start:+ reasoning_start = False+ yield "</think>"+ yield LENGTH_NOTIFICATION_CN if is_chinese(answer or generated_text) else LENGTH_NOTIFICATION_EN++ if reasoning_start:+ yield "</think>"# Flush any held-back (sanitized) answer content for this round.if _sanitizer is not None:@@ -2485,7 +2561,9 @@ class LiteLLMBase(ABC):args = json_repair.loads(tc.function.arguments)except Exception:args = {}- yield f"<think>Running the {tc.function.name} tool...</think>"+ yield "<think>"+ yield f"Running the {tc.function.name} tool..."+ yield "</think>"results = await asyncio.gather(*[_exec_tool(tc) for tc in tcs])# Terminal-tool short-circuit: a terminal tool alreadydiff --git a/test/unit_test/common/test_think_stream_parser.py b/test/unit_test/common/test_think_stream_parser.pyindex ec9ac654d0..25e1b46134 100644--- a/test/unit_test/common/test_think_stream_parser.py+++ b/test/unit_test/common/test_think_stream_parser.py@@ -49,40 +49,11 @@ CASES = [{"min_tokens": 16,"chunks": [- "<think>We</think>",- " need</think>",- " to</think>",- " respond</think>",- " to</think>",- " the</think>",- " user</think>",- "'s</think>",- " greeting</think>",- ' "</think>',- "hello</think>",- '".</think>',- " The</think>",- " assistant</think>",- " should</think>",- " be</think>",- " friendly</think>",- " and</think>",- " helpful</think>",- ".</think>",- " A</think>",- " simple</think>",- " greeting</think>",- " back</think>",- " is</think>",- " appropriate</think>",- ",</think>",- " perhaps</think>",- " with</think>",- " an</think>",- " offer</think>",- " of</think>",- " assistance</think>",- ".</think>",+ "<think>",+ "We need to respond to the user's greeting ",+ '"hello". The assistant should be friendly and helpful. ',+ "A simple greeting back is appropriate, perhaps with an offer of assistance.",+ "</think>","Hello","!"," How",@@ -96,6 +67,7 @@ CASES = ["expected": {"think": 'We need to respond to the user\'s greeting "hello". The assistant should be friendly and helpful. A simple greeting back is appropriate, perhaps with an offer of assistance.',"answer": "Hello! How can I assist you today?",+ "markers": ["<think>", "</think>"],},},),@@ -104,60 +76,13 @@ CASES = [{"min_tokens": 16,"chunks": [- "<think>We</think>",- " need</think>",- " to</think>",- " respond</think>",- " to</think>",- " the</think>",- " user</think>",- "'s</think>",- ' "</think>',- "hello</think>",- '"</think>',- " again</think>",- ".</think>",- " The</think>",- " user</think>",- " just</think>",- " said</think>",- ' "</think>',- "hello</think>",- '"</think>',- " after</think>",- " I</think>",- " already</think>",- " responded</think>",- ".</think>",- " Possibly</think>",- " they</think>",- "'re</think>",- " testing</think>",- " or</think>",- " just</think>",- " greeting</think>",- " again</think>",- ".</think>",- " I</think>",- "'ll</think>",- " respond</think>",- " in</think>",- " a</think>",- " friendly</think>",- " manner</think>",- ",</think>",- " perhaps</think>",- " acknowledging</think>",- " the</think>",- " repeated</think>",- " greeting</think>",- " and</think>",- " inviting</think>",- " them</think>",- " to</think>",- " ask</think>",- " something</think>",- ".</think>",+ "<think>",+ 'We need to respond to the user\'s "hello" again. ',+ 'The user just said "hello" after I already responded. ',+ "Possibly they're testing or just greeting again. ",+ "I'll respond in a friendly manner, perhaps acknowledging the repeated greeting ",+ "and inviting them to ask something.",+ "</think>","Hello"," again","!",@@ -172,6 +97,7 @@ CASES = ["expected": {"think": 'We need to respond to the user\'s "hello" again. The user just said "hello" after I already responded. Possibly they\'re testing or just greeting again. I\'ll respond in a friendly manner, perhaps acknowledging the repeated greeting and inviting them to ask something.',"answer": "Hello again! How can I help you today?",+ "markers": ["<think>", "</think>"],},},),@@ -266,7 +192,7 @@ CASES = ["expected": {"think": "重复开始","answer": "答案尾巴",- "markers": ["<think>", "</think>", "</think>"],+ "markers": ["<think>", "</think>"],},},),diff --git a/test/unit_test/rag/llm/test_chat_model_reasoning_stream.py b/test/unit_test/rag/llm/test_chat_model_reasoning_stream.pynew file mode 100644index 0000000000..f9b56b872f--- /dev/null+++ b/test/unit_test/rag/llm/test_chat_model_reasoning_stream.py@@ -0,0 +1,154 @@+#+# Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+#++from types import SimpleNamespace++import pytest++from rag.llm import SupportedLiteLLMProvider, chat_model+from rag.llm.chat_model import LENGTH_NOTIFICATION_CN, LENGTH_NOTIFICATION_EN, LiteLLMBase++pytestmark = pytest.mark.p1+++def _stream_chunk(*, content="", reasoning=None, finish_reason=None):+ return SimpleNamespace(+ choices=[+ SimpleNamespace(+ delta=SimpleNamespace(content=content, reasoning_content=reasoning, reasoning=None, tool_calls=None),+ finish_reason=finish_reason,+ )+ ],+ usage=None,+ )+++def _stream(chunks):+ async def _iterate():+ for chunk in chunks:+ yield chunk++ return _iterate()+++def _make_model():+ model = LiteLLMBase.__new__(LiteLLMBase)+ model.model_name = "test-model"+ model.provider = SupportedLiteLLMProvider.OpenAI+ model.api_key = "test-key"+ model.base_url = "https://example.test/v1"+ model.max_retries = 0+ model.max_rounds = 0+ model.timeout = 1+ model.tools = [+ {+ "type": "function",+ "function": {+ "name": "lookup",+ "parameters": {"type": "object", "properties": {}},+ },+ }+ ]+ model.last_usage = {"prompt_tokens": 0, "completion_tokens": 0, "total_tokens": 0}+ return model+++async def _collect(monkeypatch, model, *, chunks, with_tools=False, with_reasoning=True):+ async def fake_acompletion(**_kwargs):+ return _stream(chunks)++ monkeypatch.setattr(chat_model.litellm, "acompletion", fake_acompletion)+ history = [{"role": "user", "content": "hello"}]+ if with_tools:+ return [event async for event in model.async_chat_streamly_with_tools("", history, {})]+ return [event async for event in model.async_chat_streamly("", history, {}, with_reasoning=with_reasoning)]++[email protected]("with_tools", [False, True])+async def test_reasoning_stream_emits_one_marker_pair(monkeypatch, with_tools):+ events = await _collect(+ monkeypatch,+ _make_model(),+ with_tools=with_tools,+ chunks=[+ _stream_chunk(reasoning="reasoning one"),+ _stream_chunk(),+ _stream_chunk(reasoning=" and two"),+ _stream_chunk(content="answer", finish_reason="stop"),+ ],+ )++ text_events = [event for event in events if isinstance(event, str) and event]+ assert text_events == ["<think>", "reasoning one", " and two", "</think>", "answer"]+ assert isinstance(events[-1], int)++[email protected]("with_tools", [False, True])+async def test_reasoning_and_answer_in_same_delta_are_both_emitted(monkeypatch, with_tools):+ events = await _collect(+ monkeypatch,+ _make_model(),+ with_tools=with_tools,+ chunks=[_stream_chunk(reasoning="reasoning", content="answer", finish_reason="stop")],+ )++ text_events = [event for event in events if isinstance(event, str) and event]+ assert text_events == ["<think>", "reasoning", "</think>", "answer"]+++async def test_reasoning_stream_can_hide_reasoning(monkeypatch):+ events = await _collect(+ monkeypatch,+ _make_model(),+ with_reasoning=False,+ chunks=[+ _stream_chunk(reasoning="hidden"),+ _stream_chunk(content="answer", finish_reason="stop"),+ ],+ )++ assert [event for event in events if isinstance(event, str) and event] == ["answer"]+ assert isinstance(events[-1], int)++[email protected]("finish_reason", [None, "length"])+async def test_reasoning_only_stream_is_closed(monkeypatch, finish_reason):+ events = await _collect(monkeypatch, _make_model(), chunks=[_stream_chunk(reasoning="思考", finish_reason=finish_reason)])++ assert events[:3] == ["<think>", "思考", "</think>"]+ assert events.count("<think>") == 1+ assert events.count("</think>") == 1+ if finish_reason == "length":+ assert LENGTH_NOTIFICATION_CN in events+ assert isinstance(events[-1], int)++[email protected]("with_tools", [False, True])+async def test_length_notification_uses_answer_language(monkeypatch, with_tools):+ events = await _collect(+ monkeypatch,+ _make_model(),+ with_tools=with_tools,+ chunks=[_stream_chunk(content="这是答案", finish_reason="length")],+ )++ assert LENGTH_NOTIFICATION_CN in events+ assert LENGTH_NOTIFICATION_EN not in events