Skip to content

armgit / ECC

Public Imported from GitHub
0

The agent harness performance optimization system. Skills, instincts, memory, security, and research-first development for Claude Code, Codex, Opencode, Cursor and beyond.

fix(llm): only strip close-only reasoning when the deployment declares it

FenjuFu2861dee17672
diff --git a/src/llm/providers/ollama.py b/src/llm/providers/ollama.py
index 1cfc5120a..6ee4aa8d4 100644
--- a/src/llm/providers/ollama.py
+++ b/src/llm/providers/ollama.py
@@ -19,7 +19,7 @@ from llm.core.types import (
ProviderType,
ToolCall,
)
-from llm.providers.reasoning import strip_reasoning
+from llm.providers.reasoning import prefilled_reasoning_from_env, strip_reasoning
class OllamaProvider(LLMProvider):
@@ -29,7 +29,15 @@ class OllamaProvider(LLMProvider):
self,
base_url: str | None = None,
default_model: str | None = None,
+ prefilled_reasoning: bool | None = None,
) -> None:
+ # Set for models whose chat template prefills <think>, so replies arrive
+ # as "...</think>answer" without the opening tag.
+ self.prefilled_reasoning = (
+ prefilled_reasoning
+ if prefilled_reasoning is not None
+ else prefilled_reasoning_from_env("OLLAMA_PREFILLED_REASONING")
+ )
self.base_url = base_url or os.environ.get("OLLAMA_BASE_URL", "http://localhost:11434")
self.default_model = default_model or os.environ.get("OLLAMA_MODEL", "llama3.2")
self._models = [
@@ -86,7 +94,9 @@ class OllamaProvider(LLMProvider):
with urllib.request.urlopen(req, timeout=60) as response:
result = json.loads(response.read().decode("utf-8"))
- content = strip_reasoning(result.get("message", {}).get("content", ""))
+ content = strip_reasoning(
+ result.get("message", {}).get("content", ""), prefilled=self.prefilled_reasoning
+ )
tool_calls = None
if result.get("message", {}).get("tool_calls"):
diff --git a/src/llm/providers/openai.py b/src/llm/providers/openai.py
index 22ab043ea..011c71fcf 100644
--- a/src/llm/providers/openai.py
+++ b/src/llm/providers/openai.py
@@ -22,13 +22,25 @@ from llm.core.types import (
ToolCall,
)
from llm.providers.constants import EMPTY_FILTERED_RESPONSE_ERROR
-from llm.providers.reasoning import strip_reasoning
+from llm.providers.reasoning import prefilled_reasoning_from_env, strip_reasoning
class OpenAIProvider(LLMProvider):
provider_type = ProviderType.OPENAI
- def __init__(self, api_key: str | None = None, base_url: str | None = None) -> None:
+ def __init__(
+ self,
+ api_key: str | None = None,
+ base_url: str | None = None,
+ prefilled_reasoning: bool | None = None,
+ ) -> None:
+ # Set for self-hosted servers whose chat template prefills <think>, so
+ # replies arrive as "...</think>answer" without the opening tag.
+ self.prefilled_reasoning = (
+ prefilled_reasoning
+ if prefilled_reasoning is not None
+ else prefilled_reasoning_from_env("OPENAI_PREFILLED_REASONING")
+ )
self.client = OpenAI(
api_key=api_key or os.environ.get("OPENAI_API_KEY"),
base_url=base_url,
@@ -106,7 +118,7 @@ class OpenAIProvider(LLMProvider):
}
return LLMOutput(
- content=strip_reasoning(choice.message.content or ""),
+ content=strip_reasoning(choice.message.content or "", prefilled=self.prefilled_reasoning),
tool_calls=tool_calls,
model=response.model,
usage=usage,
diff --git a/src/llm/providers/reasoning.py b/src/llm/providers/reasoning.py
index 497ab8cf4..4f2add4ef 100644
--- a/src/llm/providers/reasoning.py
+++ b/src/llm/providers/reasoning.py
@@ -2,32 +2,48 @@
from __future__ import annotations
+import os
+
THINK_OPEN = "<think>"
THINK_CLOSE = "</think>"
+_TRUTHY = {"1", "true", "yes", "on"}
+
+
+def prefilled_reasoning_from_env(name: str) -> bool:
+ """Read a ``*_PREFILLED_REASONING`` flag from the environment."""
+ return os.environ.get(name, "").strip().lower() in _TRUTHY
-def strip_reasoning(content: str) -> str:
+
+def strip_reasoning(content: str, *, prefilled: bool = False) -> str:
"""Drop a leading reasoning block from model output.
Hosted APIs return reasoning in a separate field, but self-hosted reasoning
models (vLLM or llama.cpp without a reasoning parser, Ollama models without
- a thinking template) put it in ``content``. It arrives in two shapes:
-
- - ``<think>...</think>answer``
- - ``...</think>answer`` when the chat template prefills the opening tag in
- the generation prompt, so the model only ever emits the closing tag.
-
- A ``<think>`` that does not open the output is treated as a literal and
- left alone, as is unterminated reasoning (for example when ``max_tokens``
- cut the model off before it reached an answer).
+ a thinking template) put it in ``content``.
+
+ By default only a balanced block that opens the output is removed:
+ ``<think>...</think>answer``. Anything else is returned unchanged, so an
+ answer that merely mentions the tags (including a lone or reversed closing
+ tag) is never truncated, and neither is unterminated reasoning (for example
+ when ``max_tokens`` cut the model off before it reached an answer).
+
+ ``prefilled=True`` is for deployments whose chat template prefills the
+ opening tag in the generation prompt, so the model only ever emits
+ ``...</think>answer``. Only enable it when the model is known to behave this
+ way: everything up to the first closing tag is then treated as reasoning.
"""
- close = content.find(THINK_CLOSE)
- if close == -1:
+ stripped = content.lstrip()
+ if stripped.startswith(THINK_OPEN):
+ close = stripped.find(THINK_CLOSE, len(THINK_OPEN))
+ if close == -1:
+ return content
+ return stripped[close + len(THINK_CLOSE) :].lstrip()
+
+ if not prefilled:
return content
- head = content[:close]
- opening = head.find(THINK_OPEN)
- if opening != -1 and head[:opening].strip():
+ close = content.find(THINK_CLOSE)
+ if close == -1 or THINK_OPEN in content[:close]:
return content
-
return content[close + len(THINK_CLOSE) :].lstrip()
diff --git a/tests/test_reasoning.py b/tests/test_reasoning.py
index 8ff274908..5ac45563d 100644
--- a/tests/test_reasoning.py
+++ b/tests/test_reasoning.py
@@ -1,3 +1,4 @@
+import json
import urllib.request
from io import BytesIO
from types import SimpleNamespace
@@ -5,30 +6,63 @@ from types import SimpleNamespace
import pytest
from llm.core.types import LLMInput, Message, Role
+from llm.providers.astraflow import AstraflowProvider
from llm.providers.ollama import OllamaProvider
from llm.providers.openai import OpenAIProvider
from llm.providers.reasoning import strip_reasoning
+# Answers that mention the tags without carrying reasoning. None of them may be
+# changed, whatever the deployment's reasoning format.
+LITERAL_ANSWERS = [
+ "Wrap it in <think> and </think> tags.",
+ "Close it with </think> and nothing else.",
+ "</think> comes before <think> here.",
+ '{"closing":"</think>","opening":"<think>"}',
+ '{"note": "<think>x</think>"}',
+]
+
@pytest.mark.parametrize(
("content", "expected"),
[
("<think>17 * 3 = 51.</think>51", "51"),
("\n<think>\nplan\n</think>\n\nanswer", "answer"),
- ("17 * 3 = 51.</think>51", "51"),
+ ('<think>build the object</think>{"answer": 51}', '{"answer": 51}'),
+ ('<think>plan</think>{"closing":"</think>"}', '{"closing":"</think>"}'),
("plain answer", "plain answer"),
("", ""),
- ("Wrap it in <think> and </think> tags.", "Wrap it in <think> and </think> tags."),
("<think>cut off by max_tokens", "<think>cut off by max_tokens"),
+ # Close-only output is ambiguous without a declared format, so it stays.
+ ("17 * 3 = 51.</think>51", "17 * 3 = 51.</think>51"),
],
)
-def test_strip_reasoning(content, expected):
+def test_strip_reasoning_default(content, expected):
assert strip_reasoning(content) == expected
-def test_openai_provider_strips_inline_reasoning():
- provider = OpenAIProvider(api_key="test")
- message = SimpleNamespace(content="<think>17 * 3 = 51.</think>51", tool_calls=None)
[email protected]("content", LITERAL_ANSWERS)
+def test_strip_reasoning_default_preserves_literal_tags(content):
+ assert strip_reasoning(content) == content
+
+
+ ("content", "expected"),
+ [
+ ("17 * 3 = 51.</think>51", "51"),
+ ('build the object\n</think>\n{"answer": 51}', '{"answer": 51}'),
+ ("<think>17 * 3 = 51.</think>51", "51"),
+ ("plain answer", "plain answer"),
+ ("Wrap it in <think> and </think> tags.", "Wrap it in <think> and </think> tags."),
+ ("17 * 3 = 51.</think>", ""),
+ ],
+)
+def test_strip_reasoning_prefilled(content, expected):
+ assert strip_reasoning(content, prefilled=True) == expected
+
+
+def _openai_provider(content, **kwargs):
+ provider = OpenAIProvider(api_key="test", **kwargs)
+ message = SimpleNamespace(content=content, tool_calls=None)
response = SimpleNamespace(
choices=[SimpleNamespace(message=message, finish_reason="stop")],
model="spark-x2.5",
@@ -37,18 +71,83 @@ def test_openai_provider_strips_inline_reasoning():
provider.client = SimpleNamespace(
chat=SimpleNamespace(completions=SimpleNamespace(create=lambda **params: response))
)
+ return provider
+
+
+def _ask(provider, **kwargs):
+ return provider.generate(LLMInput(messages=[Message(role=Role.USER, content="17*3?")], **kwargs))
+
+
+def test_openai_provider_strips_balanced_reasoning():
+ assert _ask(_openai_provider("<think>17 * 3 = 51.</think>51")).content == "51"
+
+
[email protected]("content", LITERAL_ANSWERS)
+def test_openai_provider_keeps_literal_tags(content):
+ assert _ask(_openai_provider(content)).content == content
+
+
+def test_openai_provider_close_only_needs_opt_in(monkeypatch):
+ monkeypatch.delenv("OPENAI_PREFILLED_REASONING", raising=False)
+ assert _ask(_openai_provider("17 * 3 = 51.</think>51")).content == "17 * 3 = 51.</think>51"
+ assert _ask(_openai_provider("17 * 3 = 51.</think>51", prefilled_reasoning=True)).content == "51"
+
+ monkeypatch.setenv("OPENAI_PREFILLED_REASONING", "true")
+ assert _ask(_openai_provider("17 * 3 = 51.</think>51")).content == "51"
+
+
+def test_hosted_provider_never_strips_close_only(monkeypatch):
+ monkeypatch.setenv("OPENAI_PREFILLED_REASONING", "true")
+ provider = AstraflowProvider(api_key="test")
+ message = SimpleNamespace(content='{"closing":"</think>"}', tool_calls=None)
+ response = SimpleNamespace(
+ choices=[SimpleNamespace(message=message, finish_reason="stop")],
+ model="deepseek-v3.2",
+ usage=None,
+ )
+ provider.client = SimpleNamespace(
+ chat=SimpleNamespace(completions=SimpleNamespace(create=lambda **params: response))
+ )
- output = provider.generate(LLMInput(messages=[Message(role=Role.USER, content="17*3?")]))
+ assert _ask(provider).content == '{"closing":"</think>"}'
- assert output.content == "51"
+def _fake_ollama(monkeypatch, content):
+ sent = []
-def test_ollama_provider_strips_prefilled_reasoning(monkeypatch):
def fake_urlopen(request, timeout):
- return BytesIO(b'{"message": {"content": "17 * 3 = 51.</think>51"}, "done_reason": "stop"}')
+ sent.append(json.loads(request.data))
+ return BytesIO(json.dumps({"message": {"content": content}, "done_reason": "stop"}).encode())
monkeypatch.setattr(urllib.request, "urlopen", fake_urlopen)
+ return sent
+
+
[email protected]("content", LITERAL_ANSWERS)
+def test_ollama_provider_keeps_literal_tags(monkeypatch, content):
+ monkeypatch.delenv("OLLAMA_PREFILLED_REASONING", raising=False)
+ _fake_ollama(monkeypatch, content)
+
+ assert _ask(OllamaProvider()).content == content
+
+
+def test_ollama_provider_strips_prefilled_reasoning_when_declared(monkeypatch):
+ monkeypatch.delenv("OLLAMA_PREFILLED_REASONING", raising=False)
+ _fake_ollama(monkeypatch, "17 * 3 = 51.</think>51")
+ assert _ask(OllamaProvider()).content == "17 * 3 = 51.</think>51"
+ assert _ask(OllamaProvider(prefilled_reasoning=True)).content == "51"
+
+ monkeypatch.setenv("OLLAMA_PREFILLED_REASONING", "1")
+ assert _ask(OllamaProvider()).content == "51"
+
+
+def test_ollama_stream_flag_still_strips_complete_message(monkeypatch):
+ # The provider always requests a single complete message, even when the
+ # caller asks for streaming, so stripping sees the whole reply. If chunked
+ # streaming is added, this needs an incremental stripper instead.
+ sent = _fake_ollama(monkeypatch, '<think>plan</think>{"closing":"</think>"}')
- output = OllamaProvider().generate(LLMInput(messages=[Message(role=Role.USER, content="17*3?")]))
+ output = _ask(OllamaProvider(), stream=True)
- assert output.content == "51"
+ assert sent[0]["stream"] is False
+ assert output.content == '{"closing":"</think>"}'