The agent harness performance optimization system. Skills, instincts, memory, security, and research-first development for Claude Code, Codex, Opencode, Cursor and beyond.
fix(llm): only strip close-only reasoning when the deployment declares it
FenjuFu2861dee17672
diff --git a/src/llm/providers/ollama.py b/src/llm/providers/ollama.pyindex 1cfc5120a..6ee4aa8d4 100644--- a/src/llm/providers/ollama.py+++ b/src/llm/providers/ollama.py@@ -19,7 +19,7 @@ from llm.core.types import (ProviderType,ToolCall,)-from llm.providers.reasoning import strip_reasoning+from llm.providers.reasoning import prefilled_reasoning_from_env, strip_reasoningclass OllamaProvider(LLMProvider):@@ -29,7 +29,15 @@ class OllamaProvider(LLMProvider):self,base_url: str | None = None,default_model: str | None = None,+ prefilled_reasoning: bool | None = None,) -> None:+ # Set for models whose chat template prefills <think>, so replies arrive+ # as "...</think>answer" without the opening tag.+ self.prefilled_reasoning = (+ prefilled_reasoning+ if prefilled_reasoning is not None+ else prefilled_reasoning_from_env("OLLAMA_PREFILLED_REASONING")+ )self.base_url = base_url or os.environ.get("OLLAMA_BASE_URL", "http://localhost:11434")self.default_model = default_model or os.environ.get("OLLAMA_MODEL", "llama3.2")self._models = [@@ -86,7 +94,9 @@ class OllamaProvider(LLMProvider):with urllib.request.urlopen(req, timeout=60) as response:result = json.loads(response.read().decode("utf-8"))- content = strip_reasoning(result.get("message", {}).get("content", ""))+ content = strip_reasoning(+ result.get("message", {}).get("content", ""), prefilled=self.prefilled_reasoning+ )tool_calls = Noneif result.get("message", {}).get("tool_calls"):diff --git a/src/llm/providers/openai.py b/src/llm/providers/openai.pyindex 22ab043ea..011c71fcf 100644--- a/src/llm/providers/openai.py+++ b/src/llm/providers/openai.py@@ -22,13 +22,25 @@ from llm.core.types import (ToolCall,)from llm.providers.constants import EMPTY_FILTERED_RESPONSE_ERROR-from llm.providers.reasoning import strip_reasoning+from llm.providers.reasoning import prefilled_reasoning_from_env, strip_reasoningclass OpenAIProvider(LLMProvider):provider_type = ProviderType.OPENAI- def __init__(self, api_key: str | None = None, base_url: str | None = None) -> None:+ def __init__(+ self,+ api_key: str | None = None,+ base_url: str | None = None,+ prefilled_reasoning: bool | None = None,+ ) -> None:+ # Set for self-hosted servers whose chat template prefills <think>, so+ # replies arrive as "...</think>answer" without the opening tag.+ self.prefilled_reasoning = (+ prefilled_reasoning+ if prefilled_reasoning is not None+ else prefilled_reasoning_from_env("OPENAI_PREFILLED_REASONING")+ )self.client = OpenAI(api_key=api_key or os.environ.get("OPENAI_API_KEY"),base_url=base_url,@@ -106,7 +118,7 @@ class OpenAIProvider(LLMProvider):}return LLMOutput(- content=strip_reasoning(choice.message.content or ""),+ content=strip_reasoning(choice.message.content or "", prefilled=self.prefilled_reasoning),tool_calls=tool_calls,model=response.model,usage=usage,diff --git a/src/llm/providers/reasoning.py b/src/llm/providers/reasoning.pyindex 497ab8cf4..4f2add4ef 100644--- a/src/llm/providers/reasoning.py+++ b/src/llm/providers/reasoning.py@@ -2,32 +2,48 @@from __future__ import annotations+import os+THINK_OPEN = "<think>"THINK_CLOSE = "</think>"+_TRUTHY = {"1", "true", "yes", "on"}+++def prefilled_reasoning_from_env(name: str) -> bool:+ """Read a ``*_PREFILLED_REASONING`` flag from the environment."""+ return os.environ.get(name, "").strip().lower() in _TRUTHY-def strip_reasoning(content: str) -> str:++def strip_reasoning(content: str, *, prefilled: bool = False) -> str:"""Drop a leading reasoning block from model output.Hosted APIs return reasoning in a separate field, but self-hosted reasoningmodels (vLLM or llama.cpp without a reasoning parser, Ollama models without- a thinking template) put it in ``content``. It arrives in two shapes:-- - ``<think>...</think>answer``- - ``...</think>answer`` when the chat template prefills the opening tag in- the generation prompt, so the model only ever emits the closing tag.-- A ``<think>`` that does not open the output is treated as a literal and- left alone, as is unterminated reasoning (for example when ``max_tokens``- cut the model off before it reached an answer).+ a thinking template) put it in ``content``.++ By default only a balanced block that opens the output is removed:+ ``<think>...</think>answer``. Anything else is returned unchanged, so an+ answer that merely mentions the tags (including a lone or reversed closing+ tag) is never truncated, and neither is unterminated reasoning (for example+ when ``max_tokens`` cut the model off before it reached an answer).++ ``prefilled=True`` is for deployments whose chat template prefills the+ opening tag in the generation prompt, so the model only ever emits+ ``...</think>answer``. Only enable it when the model is known to behave this+ way: everything up to the first closing tag is then treated as reasoning."""- close = content.find(THINK_CLOSE)- if close == -1:+ stripped = content.lstrip()+ if stripped.startswith(THINK_OPEN):+ close = stripped.find(THINK_CLOSE, len(THINK_OPEN))+ if close == -1:+ return content+ return stripped[close + len(THINK_CLOSE) :].lstrip()++ if not prefilled:return content- head = content[:close]- opening = head.find(THINK_OPEN)- if opening != -1 and head[:opening].strip():+ close = content.find(THINK_CLOSE)+ if close == -1 or THINK_OPEN in content[:close]:return content-return content[close + len(THINK_CLOSE) :].lstrip()diff --git a/tests/test_reasoning.py b/tests/test_reasoning.pyindex 8ff274908..5ac45563d 100644--- a/tests/test_reasoning.py+++ b/tests/test_reasoning.py@@ -1,3 +1,4 @@+import jsonimport urllib.requestfrom io import BytesIOfrom types import SimpleNamespace@@ -5,30 +6,63 @@ from types import SimpleNamespaceimport pytestfrom llm.core.types import LLMInput, Message, Role+from llm.providers.astraflow import AstraflowProviderfrom llm.providers.ollama import OllamaProviderfrom llm.providers.openai import OpenAIProviderfrom llm.providers.reasoning import strip_reasoning+# Answers that mention the tags without carrying reasoning. None of them may be+# changed, whatever the deployment's reasoning format.+LITERAL_ANSWERS = [+ "Wrap it in <think> and </think> tags.",+ "Close it with </think> and nothing else.",+ "</think> comes before <think> here.",+ '{"closing":"</think>","opening":"<think>"}',+ '{"note": "<think>x</think>"}',+]+@pytest.mark.parametrize(("content", "expected"),[("<think>17 * 3 = 51.</think>51", "51"),("\n<think>\nplan\n</think>\n\nanswer", "answer"),- ("17 * 3 = 51.</think>51", "51"),+ ('<think>build the object</think>{"answer": 51}', '{"answer": 51}'),+ ('<think>plan</think>{"closing":"</think>"}', '{"closing":"</think>"}'),("plain answer", "plain answer"),("", ""),- ("Wrap it in <think> and </think> tags.", "Wrap it in <think> and </think> tags."),("<think>cut off by max_tokens", "<think>cut off by max_tokens"),+ # Close-only output is ambiguous without a declared format, so it stays.+ ("17 * 3 = 51.</think>51", "17 * 3 = 51.</think>51"),],)-def test_strip_reasoning(content, expected):+def test_strip_reasoning_default(content, expected):assert strip_reasoning(content) == expected-def test_openai_provider_strips_inline_reasoning():- provider = OpenAIProvider(api_key="test")- message = SimpleNamespace(content="<think>17 * 3 = 51.</think>51", tool_calls=None)[email protected]("content", LITERAL_ANSWERS)+def test_strip_reasoning_default_preserves_literal_tags(content):+ assert strip_reasoning(content) == content+++ ("content", "expected"),+ [+ ("17 * 3 = 51.</think>51", "51"),+ ('build the object\n</think>\n{"answer": 51}', '{"answer": 51}'),+ ("<think>17 * 3 = 51.</think>51", "51"),+ ("plain answer", "plain answer"),+ ("Wrap it in <think> and </think> tags.", "Wrap it in <think> and </think> tags."),+ ("17 * 3 = 51.</think>", ""),+ ],+)+def test_strip_reasoning_prefilled(content, expected):+ assert strip_reasoning(content, prefilled=True) == expected+++def _openai_provider(content, **kwargs):+ provider = OpenAIProvider(api_key="test", **kwargs)+ message = SimpleNamespace(content=content, tool_calls=None)response = SimpleNamespace(choices=[SimpleNamespace(message=message, finish_reason="stop")],model="spark-x2.5",@@ -37,18 +71,83 @@ def test_openai_provider_strips_inline_reasoning():provider.client = SimpleNamespace(chat=SimpleNamespace(completions=SimpleNamespace(create=lambda **params: response)))+ return provider+++def _ask(provider, **kwargs):+ return provider.generate(LLMInput(messages=[Message(role=Role.USER, content="17*3?")], **kwargs))+++def test_openai_provider_strips_balanced_reasoning():+ assert _ask(_openai_provider("<think>17 * 3 = 51.</think>51")).content == "51"++[email protected]("content", LITERAL_ANSWERS)+def test_openai_provider_keeps_literal_tags(content):+ assert _ask(_openai_provider(content)).content == content+++def test_openai_provider_close_only_needs_opt_in(monkeypatch):+ monkeypatch.delenv("OPENAI_PREFILLED_REASONING", raising=False)+ assert _ask(_openai_provider("17 * 3 = 51.</think>51")).content == "17 * 3 = 51.</think>51"+ assert _ask(_openai_provider("17 * 3 = 51.</think>51", prefilled_reasoning=True)).content == "51"++ monkeypatch.setenv("OPENAI_PREFILLED_REASONING", "true")+ assert _ask(_openai_provider("17 * 3 = 51.</think>51")).content == "51"+++def test_hosted_provider_never_strips_close_only(monkeypatch):+ monkeypatch.setenv("OPENAI_PREFILLED_REASONING", "true")+ provider = AstraflowProvider(api_key="test")+ message = SimpleNamespace(content='{"closing":"</think>"}', tool_calls=None)+ response = SimpleNamespace(+ choices=[SimpleNamespace(message=message, finish_reason="stop")],+ model="deepseek-v3.2",+ usage=None,+ )+ provider.client = SimpleNamespace(+ chat=SimpleNamespace(completions=SimpleNamespace(create=lambda **params: response))+ )- output = provider.generate(LLMInput(messages=[Message(role=Role.USER, content="17*3?")]))+ assert _ask(provider).content == '{"closing":"</think>"}'- assert output.content == "51"+def _fake_ollama(monkeypatch, content):+ sent = []-def test_ollama_provider_strips_prefilled_reasoning(monkeypatch):def fake_urlopen(request, timeout):- return BytesIO(b'{"message": {"content": "17 * 3 = 51.</think>51"}, "done_reason": "stop"}')+ sent.append(json.loads(request.data))+ return BytesIO(json.dumps({"message": {"content": content}, "done_reason": "stop"}).encode())monkeypatch.setattr(urllib.request, "urlopen", fake_urlopen)+ return sent++[email protected]("content", LITERAL_ANSWERS)+def test_ollama_provider_keeps_literal_tags(monkeypatch, content):+ monkeypatch.delenv("OLLAMA_PREFILLED_REASONING", raising=False)+ _fake_ollama(monkeypatch, content)++ assert _ask(OllamaProvider()).content == content+++def test_ollama_provider_strips_prefilled_reasoning_when_declared(monkeypatch):+ monkeypatch.delenv("OLLAMA_PREFILLED_REASONING", raising=False)+ _fake_ollama(monkeypatch, "17 * 3 = 51.</think>51")+ assert _ask(OllamaProvider()).content == "17 * 3 = 51.</think>51"+ assert _ask(OllamaProvider(prefilled_reasoning=True)).content == "51"++ monkeypatch.setenv("OLLAMA_PREFILLED_REASONING", "1")+ assert _ask(OllamaProvider()).content == "51"+++def test_ollama_stream_flag_still_strips_complete_message(monkeypatch):+ # The provider always requests a single complete message, even when the+ # caller asks for streaming, so stripping sees the whole reply. If chunked+ # streaming is added, this needs an incremental stripper instead.+ sent = _fake_ollama(monkeypatch, '<think>plan</think>{"closing":"</think>"}')- output = OllamaProvider().generate(LLMInput(messages=[Message(role=Role.USER, content="17*3?")]))+ output = _ask(OllamaProvider(), stream=True)- assert output.content == "51"+ assert sent[0]["stream"] is False+ assert output.content == '{"closing":"</think>"}'