Skip to content

Use Claude Code, Codex, VSCode, Pi, and OpenCode (and 6 other harnesses) for free (1.3B+ free tokens) from your terminal, app, IDE, or phone, and now from the browser with native browser sessions (multi-harness + multi-model) like OpenClaw (voice supported + ToS friendly)

Python

fix: replay reasoning_content for DeepSeek/NIM and expand provider smoke

Alishahryar12d2bf3de70d9
diff --git a/.env.example b/.env.example
index e9e1990a..179fddbe 100644
--- a/.env.example
+++ b/.env.example
@@ -31,6 +31,16 @@ MODEL_HAIKU=
MODEL="nvidia_nim/z-ai/glm4.7"
+# Optional live smoke model overrides. Provider smoke runs once per configured
+# provider even when MODEL/MODEL_* route to a different provider.
+FCC_SMOKE_MODEL_NVIDIA_NIM=
+FCC_SMOKE_MODEL_OPEN_ROUTER=
+FCC_SMOKE_MODEL_DEEPSEEK=
+FCC_SMOKE_MODEL_LMSTUDIO=
+FCC_SMOKE_MODEL_LLAMACPP=
+FCC_SMOKE_MODEL_OLLAMA=
+
+
# Thinking output
# Per-Claude-model switches for provider reasoning requests and Claude thinking blocks.
# Blank per-model switches inherit ENABLE_MODEL_THINKING.
diff --git a/core/anthropic/__init__.py b/core/anthropic/__init__.py
index eaea8a1c..c4bf7af2 100644
--- a/core/anthropic/__init__.py
+++ b/core/anthropic/__init__.py
@@ -4,6 +4,7 @@ from .content import extract_text_from_content, get_block_attr, get_block_type
from .conversion import (
AnthropicToOpenAIConverter,
OpenAIConversionError,
+ ReasoningReplayMode,
build_base_request_body,
)
from .errors import (
@@ -26,6 +27,7 @@ __all__ = [
"ContentType",
"HeuristicToolParser",
"OpenAIConversionError",
+ "ReasoningReplayMode",
"SSEBuilder",
"ThinkTagParser",
"append_request_id",
diff --git a/core/anthropic/conversion.py b/core/anthropic/conversion.py
index c484cebc..94eba435 100644
--- a/core/anthropic/conversion.py
+++ b/core/anthropic/conversion.py
@@ -1,6 +1,7 @@
"""Message and tool format converters."""
import json
+from enum import StrEnum
from typing import Any
from pydantic import BaseModel
@@ -13,6 +14,14 @@ class OpenAIConversionError(Exception):
"""Raised when Anthropic content cannot be converted to OpenAI chat without data loss."""
+class ReasoningReplayMode(StrEnum):
+ """How assistant reasoning history is replayed to OpenAI-compatible providers."""
+
+ DISABLED = "disabled"
+ THINK_TAGS = "think_tags"
+ REASONING_CONTENT = "reasoning_content"
+
+
def _openai_reject_native_only_top_level_fields(request_data: Any) -> None:
"""OpenAI chat providers may only convert known top-level request fields.
@@ -63,6 +72,16 @@ def _serialize_tool_result_content(tool_content: Any) -> str:
return str(tool_content)
+def _clean_reasoning_content(value: Any) -> str | None:
+ if not isinstance(value, str):
+ return None
+ return value if value else None
+
+
+def _think_tag_content(reasoning: str) -> str:
+ return f"<think>\n{reasoning}\n</think>"
+
+
class AnthropicToOpenAIConverter:
"""Convert Anthropic message format to OpenAI-compatible format."""
@@ -70,24 +89,35 @@ class AnthropicToOpenAIConverter:
def convert_messages(
messages: list[Any],
*,
- include_thinking: bool = True,
- include_reasoning_content: bool = False,
+ reasoning_replay: ReasoningReplayMode = ReasoningReplayMode.THINK_TAGS,
) -> list[dict[str, Any]]:
result = []
for msg in messages:
role = msg.role
content = msg.content
+ reasoning_content = _clean_reasoning_content(
+ getattr(msg, "reasoning_content", None)
+ )
if isinstance(content, str):
- result.append({"role": role, "content": content})
+ converted = {"role": role, "content": content}
+ if role == "assistant" and reasoning_content:
+ if reasoning_replay == ReasoningReplayMode.REASONING_CONTENT:
+ converted["reasoning_content"] = reasoning_content
+ elif reasoning_replay == ReasoningReplayMode.THINK_TAGS:
+ content_parts = [_think_tag_content(reasoning_content)]
+ if content:
+ content_parts.append(content)
+ converted["content"] = "\n\n".join(content_parts)
+ result.append(converted)
elif isinstance(content, list):
if role == "assistant":
result.extend(
AnthropicToOpenAIConverter._convert_assistant_message(
content,
- include_thinking=include_thinking,
- include_reasoning_content=include_reasoning_content,
+ reasoning_content=reasoning_content,
+ reasoning_replay=reasoning_replay,
)
)
elif role == "user":
@@ -103,8 +133,8 @@ class AnthropicToOpenAIConverter:
def _convert_assistant_message(
content: list[Any],
*,
- include_thinking: bool = True,
- include_reasoning_content: bool = False,
+ reasoning_content: str | None = None,
+ reasoning_replay: ReasoningReplayMode = ReasoningReplayMode.THINK_TAGS,
) -> list[dict[str, Any]]:
content_parts: list[str] = []
thinking_parts: list[str] = []
@@ -123,7 +153,7 @@ class AnthropicToOpenAIConverter:
)
content_parts.append(get_block_attr(block, "text", ""))
elif block_type == "thinking":
- if not include_thinking:
+ if reasoning_replay == ReasoningReplayMode.DISABLED:
continue
if seen_tool_use:
raise OpenAIConversionError(
@@ -132,8 +162,9 @@ class AnthropicToOpenAIConverter:
"native Anthropic provider."
)
thinking = get_block_attr(block, "thinking", "")
- content_parts.append(f"<think>\n{thinking}\n</think>")
- if include_reasoning_content:
+ if reasoning_replay == ReasoningReplayMode.THINK_TAGS:
+ content_parts.append(_think_tag_content(thinking))
+ elif reasoning_content is None:
thinking_parts.append(thinking)
elif block_type == "redacted_thinking":
# Opaque provider continuation data; do not materialize as model-visible text
@@ -178,8 +209,10 @@ class AnthropicToOpenAIConverter:
}
if tool_calls:
msg["tool_calls"] = tool_calls
- if include_reasoning_content and thinking_parts:
- msg["reasoning_content"] = "\n".join(thinking_parts)
+ if reasoning_replay == ReasoningReplayMode.REASONING_CONTENT:
+ replay_reasoning = reasoning_content or "\n".join(thinking_parts)
+ if replay_reasoning:
+ msg["reasoning_content"] = replay_reasoning
return [msg]
@@ -271,15 +304,13 @@ def build_base_request_body(
request_data: Any,
*,
default_max_tokens: int | None = None,
- include_thinking: bool = True,
- include_reasoning_content: bool = False,
+ reasoning_replay: ReasoningReplayMode = ReasoningReplayMode.THINK_TAGS,
) -> dict[str, Any]:
"""Build the common parts of an OpenAI-format request body."""
_openai_reject_native_only_top_level_fields(request_data)
messages = AnthropicToOpenAIConverter.convert_messages(
request_data.messages,
- include_thinking=include_thinking,
- include_reasoning_content=include_reasoning_content,
+ reasoning_replay=reasoning_replay,
)
system = getattr(request_data, "system", None)
diff --git a/providers/deepseek/request.py b/providers/deepseek/request.py
index f22a9f81..057a9a46 100644
--- a/providers/deepseek/request.py
+++ b/providers/deepseek/request.py
@@ -4,7 +4,7 @@ from typing import Any
from loguru import logger
-from core.anthropic import build_base_request_body
+from core.anthropic import ReasoningReplayMode, build_base_request_body
from core.anthropic.conversion import OpenAIConversionError
from providers.exceptions import InvalidRequestError
@@ -19,8 +19,9 @@ def build_request_body(request_data: Any, *, thinking_enabled: bool) -> dict:
try:
body = build_base_request_body(
request_data,
- include_thinking=thinking_enabled,
- include_reasoning_content=thinking_enabled,
+ reasoning_replay=ReasoningReplayMode.REASONING_CONTENT
+ if thinking_enabled
+ else ReasoningReplayMode.DISABLED,
)
except OpenAIConversionError as exc:
raise InvalidRequestError(str(exc)) from exc
diff --git a/providers/nvidia_nim/client.py b/providers/nvidia_nim/client.py
index 26612030..ef06c108 100644
--- a/providers/nvidia_nim/client.py
+++ b/providers/nvidia_nim/client.py
@@ -15,6 +15,7 @@ from .request import (
build_request_body,
clone_body_without_chat_template,
clone_body_without_reasoning_budget,
+ clone_body_without_reasoning_content,
)
@@ -68,4 +69,13 @@ class NvidiaNimProvider(OpenAIChatTransport):
logger.warning("NIM_STREAM: retrying without chat_template after 400 error")
return retry_body
+ if "reasoning_content" in error_text:
+ retry_body = clone_body_without_reasoning_content(body)
+ if retry_body is None:
+ return None
+ logger.warning(
+ "NIM_STREAM: retrying without reasoning_content after 400 error"
+ )
+ return retry_body
+
return None
diff --git a/providers/nvidia_nim/request.py b/providers/nvidia_nim/request.py
index a4e96deb..8494bb5d 100644
--- a/providers/nvidia_nim/request.py
+++ b/providers/nvidia_nim/request.py
@@ -7,7 +7,11 @@ from typing import Any
from loguru import logger
from config.nim import NimSettings
-from core.anthropic import build_base_request_body, set_if_not_none
+from core.anthropic import (
+ ReasoningReplayMode,
+ build_base_request_body,
+ set_if_not_none,
+)
from core.anthropic.conversion import OpenAIConversionError
from providers.exceptions import InvalidRequestError
@@ -46,6 +50,20 @@ def _strip_chat_template_field(extra_body: dict[str, Any]) -> bool:
return extra_body.pop("chat_template", None) is not None
+def _strip_message_reasoning_content(body: dict[str, Any]) -> bool:
+ removed = False
+ messages = body.get("messages")
+ if not isinstance(messages, list):
+ return False
+ for message in messages:
+ if (
+ isinstance(message, dict)
+ and message.pop("reasoning_content", None) is not None
+ ):
+ removed = True
+ return removed
+
+
def _set_extra(
extra_body: dict[str, Any], key: str, value: Any, ignore_value: Any = None
) -> None:
@@ -68,6 +86,14 @@ def clone_body_without_chat_template(body: dict[str, Any]) -> dict[str, Any] | N
return _clone_strip_extra_body(body, _strip_chat_template_field)
+def clone_body_without_reasoning_content(body: dict[str, Any]) -> dict[str, Any] | None:
+ """Clone a request body and strip assistant message ``reasoning_content`` fields."""
+ cloned_body = deepcopy(body)
+ if not _strip_message_reasoning_content(cloned_body):
+ return None
+ return cloned_body
+
+
def build_request_body(
request_data: Any, nim: NimSettings, *, thinking_enabled: bool
) -> dict:
@@ -80,7 +106,9 @@ def build_request_body(
try:
body = build_base_request_body(
request_data,
- include_thinking=thinking_enabled,
+ reasoning_replay=ReasoningReplayMode.REASONING_CONTENT
+ if thinking_enabled
+ else ReasoningReplayMode.DISABLED,
)
except OpenAIConversionError as exc:
raise InvalidRequestError(str(exc)) from exc
diff --git a/smoke/README.md b/smoke/README.md
index db5af1da..775ebcc1 100644
--- a/smoke/README.md
+++ b/smoke/README.md
@@ -31,9 +31,11 @@ $env:FCC_LIVE_SMOKE = "1"
uv run pytest smoke -n 0 -s --tb=short
```
-Provider product E2E runs for every configured provider model from `MODEL`,
-`MODEL_OPUS`, `MODEL_SONNET`, and `MODEL_HAIKU`. If no provider is configured,
-live product smoke fails as `missing_env` unless you explicitly set
+Provider product E2E runs once per configured provider, independent of `MODEL`,
+`MODEL_OPUS`, `MODEL_SONNET`, and `MODEL_HAIKU`. Defaults come from the provider
+catalog/docs and can be overridden with `FCC_SMOKE_MODEL_<PROVIDER>`, for example
+`FCC_SMOKE_MODEL_DEEPSEEK=deepseek-reasoner`. If no provider smoke model is
+configured, live product smoke fails as `missing_env` unless you explicitly set
`FCC_ALLOW_NO_PROVIDER_SMOKE=1`.
## Targets
@@ -49,7 +51,7 @@ Default targets do not send real bot messages or load voice backends:
| `config` | env precedence, removed-env migration, proxy/timeouts | none |
| `extensibility` | provider registry and platform factory construction | none |
| `messaging` | fake Discord/Telegram full flow, commands, trees, persistence, voice cancel | none |
-| `providers` | multi-turn text, adaptive thinking history, tools, disconnect, errors | configured provider models |
+| `providers` | multi-turn text, adaptive thinking history, tools, disconnect, errors | configured providers, optional `FCC_SMOKE_MODEL_*` |
| `tools` | forced tool_use and tool_result continuation | tool-capable configured provider |
| `rate_limit` | disconnect cleanup and follow-up request | configured provider |
| `lmstudio` | local `/models` plus native `/messages` through proxy | running LM Studio server |
@@ -99,6 +101,11 @@ uv run pytest smoke/product -n 0 -s --tb=short
- `FCC_ALLOW_NO_PROVIDER_SMOKE=1`: permits no-provider live smoke for harness work.
- `FCC_SMOKE_TARGETS`: comma-separated targets, or `all`.
- `FCC_SMOKE_PROVIDER_MATRIX`: comma-separated provider prefixes to require.
+- `FCC_SMOKE_MODEL_NVIDIA_NIM`, `FCC_SMOKE_MODEL_OPEN_ROUTER`,
+ `FCC_SMOKE_MODEL_DEEPSEEK`, `FCC_SMOKE_MODEL_LMSTUDIO`,
+ `FCC_SMOKE_MODEL_LLAMACPP`, `FCC_SMOKE_MODEL_OLLAMA`: optional per-provider
+ smoke model overrides. Values may include the provider prefix or just the model
+ name for that provider.
- `FCC_SMOKE_TIMEOUT_S`: per-request/subprocess timeout, default `45`.
- `FCC_SMOKE_CLAUDE_BIN`: Claude CLI executable name, default `claude`.
- `FCC_SMOKE_TELEGRAM_CHAT_ID`: Telegram chat/user ID for send/edit/delete.
diff --git a/smoke/capabilities.py b/smoke/capabilities.py
index eafae44a..4b0a5a27 100644
--- a/smoke/capabilities.py
+++ b/smoke/capabilities.py
@@ -100,7 +100,10 @@ CAPABILITY_CONTRACTS: tuple[CapabilityContract, ...] = (
"configured BaseProvider instance",
"503 for missing credentials; invalid_request_error for unknown provider",
("tests/api/test_dependencies.py", "tests/providers/test_registry.py"),
- ("test_configured_provider_models_stream_successfully",),
+ (
+ "test_configured_provider_models_stream_successfully",
+ "test_provider_matrix_presence_e2e",
+ ),
),
CapabilityContract(
"provider_routing",
@@ -168,9 +171,15 @@ CAPABILITY_CONTRACTS: tuple[CapabilityContract, ...] = (
"thinking hidden when disabled",
(
"tests/contracts/test_stream_contracts.py",
+ "tests/providers/test_converter.py",
+ "tests/providers/test_deepseek.py",
+ "tests/providers/test_nvidia_nim_request.py",
"tests/providers/test_open_router.py",
),
- ("test_per_model_thinking_config_e2e",),
+ (
+ "test_per_model_thinking_config_e2e",
+ "test_provider_reasoning_tool_continuation_e2e",
+ ),
),
CapabilityContract(
"streaming_conversion",
@@ -181,7 +190,10 @@ CAPABILITY_CONTRACTS: tuple[CapabilityContract, ...] = (
"structured Anthropic tool_use blocks",
"text fallback when malformed",
("tests/providers/test_parsers.py", "tests/contracts/test_stream_contracts.py"),
- ("test_live_tool_use_when_configured_model_supports_tools",),
+ (
+ "test_live_tool_use_when_configured_model_supports_tools",
+ "test_provider_reasoning_tool_continuation_e2e",
+ ),
),
CapabilityContract(
"streaming_conversion",
diff --git a/smoke/features.py b/smoke/features.py
index 7d81b6de..fa0bc18b 100644
--- a/smoke/features.py
+++ b/smoke/features.py
@@ -87,7 +87,7 @@ FEATURE_INVENTORY: tuple[FeatureCoverage, ...] = (
("test_configured_provider_models_stream_successfully",),
("test_provider_matrix_presence_e2e", "test_provider_text_multiturn_e2e"),
("providers",),
- ("MODEL", "MODEL_OPUS", "MODEL_SONNET", "MODEL_HAIKU"),
+ ("configured provider credentials/endpoints", "optional FCC_SMOKE_MODEL_*"),
"selected providers missing credentials are failing missing_env",
),
FeatureCoverage(
@@ -123,6 +123,7 @@ FEATURE_INVENTORY: tuple[FeatureCoverage, ...] = (
(),
(
"test_provider_adaptive_thinking_history_e2e",
+ "test_provider_reasoning_tool_continuation_e2e",
"test_claude_cli_adaptive_thinking_e2e",
"test_per_model_thinking_config_e2e",
),
@@ -139,6 +140,7 @@ FEATURE_INVENTORY: tuple[FeatureCoverage, ...] = (
(
"test_provider_interleaved_thinking_tool_e2e",
"test_provider_tool_result_continuation_e2e",
+ "test_provider_reasoning_tool_continuation_e2e",
),
("tools", "providers"),
("configured tool-capable provider",),
diff --git a/smoke/lib/config.py b/smoke/lib/config.py
index 0f94964f..11c2afd8 100644
--- a/smoke/lib/config.py
+++ b/smoke/lib/config.py
@@ -7,6 +7,7 @@ from collections.abc import Mapping
from dataclasses import dataclass
from pathlib import Path
+from config.provider_catalog import PROVIDER_CATALOG, SUPPORTED_PROVIDER_IDS
from config.settings import Settings, get_settings
DEFAULT_TARGETS = frozenset(
@@ -36,6 +37,15 @@ TARGET_ALIASES = {
}
SECRET_KEY_PARTS = ("KEY", "TOKEN", "SECRET", "WEBHOOK", "AUTH")
+PROVIDER_SMOKE_DEFAULT_MODELS: dict[str, str] = {
+ "nvidia_nim": "nvidia_nim/z-ai/glm4.7",
+ "open_router": "open_router/stepfun/step-3.5-flash:free",
+ "deepseek": "deepseek/deepseek-chat",
+ "lmstudio": "lmstudio/local-model",
+ "llamacpp": "llamacpp/local-model",
+ "ollama": "ollama/llama3.1",
+}
+
TARGET_REQUIRED_ENV: dict[str, tuple[str, ...]] = {
"api": (),
@@ -45,7 +55,7 @@ TARGET_REQUIRED_ENV: dict[str, tuple[str, ...]] = {
"config": (),
"extensibility": (),
"messaging": (),
- "providers": ("MODEL or MODEL_* with usable provider configuration",),
+ "providers": ("configured provider credentials/endpoints or FCC_SMOKE_MODEL_*",),
"rate_limit": ("configured provider model",),
"tools": ("configured tool-capable provider model",),
"lmstudio": ("LM_STUDIO_BASE_URL with a running LM Studio server",),
@@ -133,6 +143,35 @@ class SmokeConfig:
)
return models
+ def provider_smoke_models(self) -> list[ProviderModel]:
+ """Return one smoke model per configured provider, independent of MODEL_*."""
+ models: list[ProviderModel] = []
+ mapped_providers = {model.provider for model in self.provider_models()}
+ for provider in SUPPORTED_PROVIDER_IDS:
+ if self.provider_matrix and provider not in self.provider_matrix:
+ continue
+ if not self.has_provider_configuration(provider):
+ continue
+ if not self._include_provider_in_smoke(provider, mapped_providers):
+ continue
+ full_model, source = _provider_smoke_model(provider)
+ models.append(
+ ProviderModel(provider=provider, full_model=full_model, source=source)
+ )
+ return models
+
+ def _include_provider_in_smoke(
+ self, provider: str, mapped_providers: set[str]
+ ) -> bool:
+ descriptor = PROVIDER_CATALOG[provider]
+ if "local" not in descriptor.capabilities:
+ return True
+ if provider in mapped_providers:
+ return True
+ if self.provider_matrix and provider in self.provider_matrix:
+ return True
+ return bool(os.getenv(f"FCC_SMOKE_MODEL_{provider.upper()}"))
+
def has_provider_configuration(self, provider: str) -> bool:
if provider == "nvidia_nim":
return bool(self.settings.nvidia_nim_api_key.strip())
@@ -164,6 +203,37 @@ def _parse_targets(raw: str | None) -> frozenset[str]:
return frozenset(TARGET_ALIASES.get(target, target) for target in parsed)
+def _provider_smoke_model(provider: str) -> tuple[str, str]:
+ override_env = f"FCC_SMOKE_MODEL_{provider.upper()}"
+ if override := os.getenv(override_env):
+ return _normalize_provider_model(provider, override), override_env
+
+ default = PROVIDER_SMOKE_DEFAULT_MODELS.get(provider)
+ if default is None:
+ descriptor = PROVIDER_CATALOG[provider]
+ default = f"{descriptor.provider_id}/smoke-default"
+ return default, "provider_default"
+
+
+def _normalize_provider_model(provider: str, raw_model: str) -> str:
+ model = raw_model.strip()
+ if not model:
+ msg = f"FCC_SMOKE_MODEL_{provider.upper()} must not be empty"
+ raise ValueError(msg)
+ if "/" not in model:
+ return f"{provider}/{model}"
+ prefix = Settings.parse_provider_type(model)
+ if prefix == provider:
+ return model
+ if prefix in SUPPORTED_PROVIDER_IDS:
+ msg = (
+ f"FCC_SMOKE_MODEL_{provider.upper()} must use provider prefix "
+ f"{provider!r}, got {model!r}"
+ )
+ raise ValueError(msg)
+ return f"{provider}/{model}"
+
+
def auth_headers(token: str | None = None) -> dict[str, str]:
settings = get_settings()
resolved = token if token is not None else settings.anthropic_auth_token
diff --git a/smoke/lib/e2e.py b/smoke/lib/e2e.py
index 8b20e809..dbd39cf8 100644
--- a/smoke/lib/e2e.py
+++ b/smoke/lib/e2e.py
@@ -161,6 +161,9 @@ class ProviderMatrixDriver:
self.config = config
def configured_models(self) -> list[ProviderModel]:
+ return self.config.provider_models()
+
+ def provider_smoke_models(self) -> list[ProviderModel]:
selected = self.config.provider_matrix
missing_selected = [
provider
@@ -174,16 +177,16 @@ class ProviderMatrixDriver:
+ ", ".join(sorted(missing_selected))
)
- models = self.config.provider_models()
+ models = self.config.provider_smoke_models()
if not models and os.getenv("FCC_ALLOW_NO_PROVIDER_SMOKE") != "1":
fail_missing_env(
- "no configured provider models; set FCC_ALLOW_NO_PROVIDER_SMOKE=1 "
+ "no configured provider smoke models; set FCC_ALLOW_NO_PROVIDER_SMOKE=1 "
"only for no-provider smoke collection"
)
return models
def first_model(self) -> ProviderModel:
- models = self.configured_models()
+ models = self.provider_smoke_models()
if not models:
pytest.skip("missing_env: no configured provider model")
return models[0]
diff --git a/smoke/prereq/test_provider_prereq_live.py b/smoke/prereq/test_provider_prereq_live.py
index 0d85356c..ed5d5ffd 100644
--- a/smoke/prereq/test_provider_prereq_live.py
+++ b/smoke/prereq/test_provider_prereq_live.py
@@ -10,6 +10,7 @@ from core.anthropic.stream_contracts import (
text_content,
)
from smoke.lib.config import SmokeConfig, auth_headers
+from smoke.lib.e2e import ProviderMatrixDriver
from smoke.lib.http import collect_message_stream, message_payload
from smoke.lib.server import start_server
from smoke.lib.skips import (
@@ -45,9 +46,7 @@ def test_mixed_provider_model_mapping_when_configured(
def test_configured_provider_models_stream_successfully(
smoke_config: SmokeConfig,
) -> None:
- models = smoke_config.provider_models()
- if not models:
- pytest.skip("no configured provider models with usable credentials/base URLs")
+ models = ProviderMatrixDriver(smoke_config).provider_smoke_models()
failures: list[str] = []
for provider_model in models:
@@ -82,10 +81,7 @@ def test_configured_provider_models_stream_successfully(
def test_client_disconnect_mid_stream_does_not_crash_server(
smoke_config: SmokeConfig,
) -> None:
- models = smoke_config.provider_models()
- if not models:
- pytest.skip("no configured provider model available for disconnect smoke")
- provider_model = models[0]
+ provider_model = ProviderMatrixDriver(smoke_config).first_model()
with start_server(
smoke_config,
diff --git a/smoke/product/test_provider_product_live.py b/smoke/product/test_provider_product_live.py
index be5d45e9..469163bf 100644
--- a/smoke/product/test_provider_product_live.py
+++ b/smoke/product/test_provider_product_live.py
@@ -3,6 +3,7 @@ from __future__ import annotations
import httpx
import pytest
+from config.provider_catalog import PROVIDER_CATALOG
from core.anthropic.stream_contracts import (
assert_anthropic_stream_contract,
parse_sse_lines,
@@ -26,7 +27,7 @@ pytestmark = [pytest.mark.live, pytest.mark.smoke_target("providers")]
def test_provider_matrix_presence_e2e(smoke_config: SmokeConfig) -> None:
- models = ProviderMatrixDriver(smoke_config).configured_models()
+ models = ProviderMatrixDriver(smoke_config).provider_smoke_models()
assert models or smoke_config.provider_matrix == frozenset()
@@ -56,6 +57,11 @@ def test_provider_tool_result_continuation_e2e(smoke_config: SmokeConfig) -> Non
_run_for_each_provider(smoke_config, _scenario_tool_result_continuation)
[email protected]_target("tools")
+def test_provider_reasoning_tool_continuation_e2e(smoke_config: SmokeConfig) -> None:
+ _run_for_each_thinking_provider(smoke_config, _scenario_reasoning_tool_continuation)
+
+
@pytest.mark.smoke_target("rate_limit")
def test_provider_disconnect_e2e(smoke_config: SmokeConfig) -> None:
_run_for_each_provider(smoke_config, _scenario_disconnect)
@@ -91,7 +97,7 @@ def test_provider_error_e2e(smoke_config: SmokeConfig) -> None:
def test_openrouter_native_e2e(smoke_config: SmokeConfig) -> None:
models = [
model
- for model in ProviderMatrixDriver(smoke_config).configured_models()
+ for model in ProviderMatrixDriver(smoke_config).provider_smoke_models()
if model.provider == "open_router"
]
if not models:
@@ -124,7 +130,7 @@ def test_openrouter_native_e2e(smoke_config: SmokeConfig) -> None:
def _run_for_each_provider(smoke_config: SmokeConfig, scenario) -> None:
failures: list[str] = []
- for provider_model in ProviderMatrixDriver(smoke_config).configured_models():
+ for provider_model in ProviderMatrixDriver(smoke_config).provider_smoke_models():
try:
scenario(smoke_config, provider_model)
except Exception as exc:
@@ -136,6 +142,37 @@ def _run_for_each_provider(smoke_config: SmokeConfig, scenario) -> None:
assert not failures, "\n".join(failures)
+def _run_for_each_thinking_provider(smoke_config: SmokeConfig, scenario) -> None:
+ failures: list[str] = []
+ models = [
+ provider_model
+ for provider_model in ProviderMatrixDriver(smoke_config).provider_smoke_models()
+ if _provider_smoke_thinking_enabled(smoke_config, provider_model)
+ ]
+ if not models:
+ pytest.skip("missing_env: no thinking-capable provider smoke model configured")
+ for provider_model in models:
+ try:
+ scenario(smoke_config, provider_model)
+ except Exception as exc:
+ skip_if_upstream_unavailable_exception(exc)
+ failures.append(
+ f"{provider_model.source}={provider_model.full_model}: "
+ f"{type(exc).__name__}: {exc}"
+ )
+ assert not failures, "\n".join(failures)
+
+
+def _provider_smoke_thinking_enabled(
+ smoke_config: SmokeConfig, provider_model: ProviderModel
+) -> bool:
+ descriptor = PROVIDER_CATALOG[provider_model.provider]
+ return (
+ "thinking" in descriptor.capabilities
+ and smoke_config.settings.resolve_thinking("claude-sonnet-4-5-20250929")
+ )
+
+
def _scenario_text_multiturn(
smoke_config: SmokeConfig, provider_model: ProviderModel
) -> None:
@@ -255,6 +292,45 @@ def _scenario_tool_result_continuation(
assert_product_stream(second.events)
+def _scenario_reasoning_tool_continuation(
+ smoke_config: SmokeConfig, provider_model: ProviderModel
+) -> None:
+ payload = {
+ "model": "claude-sonnet-4-5-20250929",
+ "max_tokens": 256,
+ "messages": [
+ {"role": "user", "content": "Use echo_smoke once with value FCC_TOOL."},
+ {
+ "role": "assistant",
+ "content": [
+ {"type": "thinking", "thinking": "Need to return the echo result."},
+ {
+ "type": "tool_use",
+ "id": "toolu_reasoning_smoke",
+ "name": "echo_smoke",
+ "input": {"value": "FCC_TOOL"},
+ },
+ ],
+ },
+ {
+ "role": "user",
+ "content": [
+ {
+ "type": "tool_result",
+ "tool_use_id": "toolu_reasoning_smoke",
+ "content": "FCC_TOOL",
+ }
+ ],
+ },
+ ],
+ "tools": [echo_tool_schema()],
+ "thinking": {"type": "adaptive"},
+ }
+ with _server_for_provider(smoke_config, provider_model, "reasoning-tool") as server:
+ turn = ConversationDriver(server, smoke_config).stream(payload)
+ assert_product_stream(turn.events)
+
+
def _scenario_disconnect(
smoke_config: SmokeConfig, provider_model: ProviderModel
) -> None:
diff --git a/tests/contracts/test_smoke_config.py b/tests/contracts/test_smoke_config.py
index e8a1de7b..9cd8424d 100644
--- a/tests/contracts/test_smoke_config.py
+++ b/tests/contracts/test_smoke_config.py
@@ -3,7 +3,12 @@ from __future__ import annotations
from pathlib import Path
from types import SimpleNamespace
-from smoke.lib.config import DEFAULT_TARGETS, TARGET_REQUIRED_ENV, SmokeConfig
+from smoke.lib.config import (
+ DEFAULT_TARGETS,
+ PROVIDER_SMOKE_DEFAULT_MODELS,
+ TARGET_REQUIRED_ENV,
+ SmokeConfig,
+)
def _settings(**overrides):
@@ -57,3 +62,114 @@ def test_ollama_provider_matrix_filters_models() -> None:
config = _smoke_config(provider_matrix=frozenset({"ollama"}))
assert [model.provider for model in config.provider_models()] == ["ollama"]
+
+
+def test_provider_smoke_models_cover_configured_providers_independent_of_model_mapping(
+ monkeypatch,
+) -> None:
+ monkeypatch.delenv("FCC_SMOKE_MODEL_DEEPSEEK", raising=False)
+ config = _smoke_config(
+ settings=_settings(
+ model="ollama/llama3.1",
+ deepseek_api_key="deepseek-key",
+ ollama_base_url="",
+ )
+ )
+
+ models = config.provider_smoke_models()
+
+ assert [model.provider for model in models] == ["deepseek"]
+ assert models[0].full_model == PROVIDER_SMOKE_DEFAULT_MODELS["deepseek"]
+ assert models[0].source == "provider_default"
+
+
+def test_provider_smoke_model_override_accepts_model_name_without_prefix(
+ monkeypatch,
+) -> None:
+ monkeypatch.setenv("FCC_SMOKE_MODEL_DEEPSEEK", "deepseek-reasoner")
+ config = _smoke_config(
+ settings=_settings(
+ deepseek_api_key="deepseek-key",
+ ollama_base_url="",
+ ),
+ provider_matrix=frozenset({"deepseek"}),
+ )
+
+ models = config.provider_smoke_models()
+
+ assert models[0].full_model == "deepseek/deepseek-reasoner"
+ assert models[0].source == "FCC_SMOKE_MODEL_DEEPSEEK"
+
+
+def test_provider_smoke_model_override_accepts_owner_model_name(
+ monkeypatch,
+) -> None:
+ monkeypatch.setenv("FCC_SMOKE_MODEL_NVIDIA_NIM", "z-ai/glm4.7")
+ config = _smoke_config(
+ settings=_settings(
+ model="deepseek/deepseek-chat",
+ deepseek_api_key="",
+ nvidia_nim_api_key="nim-key",
+ ollama_base_url="",
+ ),
+ provider_matrix=frozenset({"nvidia_nim"}),
+ )
+
+ models = config.provider_smoke_models()
+
+ assert models[0].full_model == "nvidia_nim/z-ai/glm4.7"
+ assert models[0].source == "FCC_SMOKE_MODEL_NVIDIA_NIM"
+
+
+def test_provider_smoke_model_override_rejects_wrong_provider_prefix(
+ monkeypatch,
+) -> None:
+ monkeypatch.setenv("FCC_SMOKE_MODEL_DEEPSEEK", "ollama/llama3.1")
+ config = _smoke_config(
+ settings=_settings(
+ deepseek_api_key="deepseek-key",
+ ollama_base_url="",
+ ),
+ provider_matrix=frozenset({"deepseek"}),
+ )
+
+ try:
+ config.provider_smoke_models()
+ except ValueError as exc:
+ assert "FCC_SMOKE_MODEL_DEEPSEEK" in str(exc)
+ else:
+ raise AssertionError("expected wrong provider prefix to fail")
+
+
+def test_provider_smoke_matrix_filters_provider_catalog(monkeypatch) -> None:
+ monkeypatch.delenv("FCC_SMOKE_MODEL_DEEPSEEK", raising=False)
+ config = _smoke_config(
+ settings=_settings(
+ deepseek_api_key="deepseek-key",
+ nvidia_nim_api_key="nim-key",
+ ollama_base_url="",
+ ),
+ provider_matrix=frozenset({"nvidia_nim"}),
+ )
+
+ assert [model.provider for model in config.provider_smoke_models()] == [
+ "nvidia_nim"
+ ]
+
+
+def test_provider_smoke_includes_local_provider_when_model_mapping_uses_it(
+ monkeypatch,
+) -> None:
+ monkeypatch.delenv("FCC_SMOKE_MODEL_OLLAMA", raising=False)
+ config = _smoke_config()
+
+ assert [model.provider for model in config.provider_smoke_models()] == ["ollama"]
+
+
+def test_provider_smoke_does_not_include_default_local_urls_when_unmapped(
+ monkeypatch,
+) -> None:
+ monkeypatch.delenv("FCC_SMOKE_MODEL_OLLAMA", raising=False)
+ config = _smoke_config(settings=_settings(model="nvidia_nim/test"))
+
+ assert config.provider_smoke_models() == []
diff --git a/tests/providers/test_converter.py b/tests/providers/test_converter.py
index 99aeba6f..a5aa6b16 100644
--- a/tests/providers/test_converter.py
+++ b/tests/providers/test_converter.py
@@ -6,6 +6,7 @@ from api.models.anthropic import MessagesRequest
from core.anthropic import (
AnthropicToOpenAIConverter,
OpenAIConversionError,
+ ReasoningReplayMode,
build_base_request_body,
)
@@ -13,9 +14,10 @@ from core.anthropic import (
class MockMessage:
- def __init__(self, role, content):
+ def __init__(self, role, content, reasoning_content=None):
self.role = role
self.content = content
+ self.reasoning_content = reasoning_content
class MockBlock:
@@ -226,20 +228,64 @@ def test_convert_assistant_message_thinking():
assert "reasoning_content" not in result[0]
-def test_convert_assistant_message_thinking_include_reasoning_content():
- """When include_reasoning_content=True, reasoning_content is added."""
+def test_convert_assistant_message_thinking_replays_reasoning_content():
+ """Top-level reasoning replay avoids duplicating thinking into content."""
content = [
MockBlock(type="thinking", thinking="I need to calculate this."),
MockBlock(type="text", text="The answer is 4."),
]
messages = [MockMessage("assistant", content)]
result = AnthropicToOpenAIConverter.convert_messages(
- messages, include_reasoning_content=True
+ messages, reasoning_replay=ReasoningReplayMode.REASONING_CONTENT
)
assert len(result) == 1
assert result[0]["reasoning_content"] == "I need to calculate this."
- assert "<think>" in result[0]["content"]
+ assert result[0]["content"] == "The answer is 4."
+ assert "<think>" not in result[0]["content"]
+
+
+def test_convert_assistant_top_level_reasoning_content_is_preserved():
+ messages = [
+ MockMessage(
+ "assistant",
+ "The answer is 4.",
+ reasoning_content="I need to calculate this.",
+ )
+ ]
+ result = AnthropicToOpenAIConverter.convert_messages(
+ messages, reasoning_replay=ReasoningReplayMode.REASONING_CONTENT
+ )
+
+ assert result == [
+ {
+ "role": "assistant",
+ "content": "The answer is 4.",
+ "reasoning_content": "I need to calculate this.",
+ }
+ ]
+
+
+def test_convert_assistant_thinking_tool_use_replays_top_level_reasoning():
+ content = [
+ MockBlock(type="thinking", thinking="I should call the tool."),
+ MockBlock(
+ type="tool_use",
+ id="call_reasoning",
+ name="search",
+ input={"query": "python"},
+ ),
+ ]
+ messages = [MockMessage("assistant", content)]
+ result = AnthropicToOpenAIConverter.convert_messages(
+ messages, reasoning_replay=ReasoningReplayMode.REASONING_CONTENT
+ )
+
+ assert len(result) == 1
+ assert result[0]["content"] == ""
+ assert result[0]["reasoning_content"] == "I should call the tool."
+ assert "<think>" not in result[0]["content"]
+ assert result[0]["tool_calls"][0]["id"] == "call_reasoning"
def test_convert_assistant_message_thinking_removed_when_disabled():
@@ -250,8 +296,7 @@ def test_convert_assistant_message_thinking_removed_when_disabled():
messages = [MockMessage("assistant", content)]
result = AnthropicToOpenAIConverter.convert_messages(
messages,
- include_thinking=False,
- include_reasoning_content=True,
+ reasoning_replay=ReasoningReplayMode.DISABLED,
)
assert len(result) == 1
@@ -260,6 +305,21 @@ def test_convert_assistant_message_thinking_removed_when_disabled():
assert result[0]["content"] == "The answer is 4."
+def test_convert_assistant_top_level_reasoning_stripped_when_disabled():
+ messages = [
+ MockMessage(
+ "assistant",
+ "The answer is 4.",
+ reasoning_content="I need to calculate this.",
+ )
+ ]
+ result = AnthropicToOpenAIConverter.convert_messages(
+ messages, reasoning_replay=ReasoningReplayMode.DISABLED
+ )
+
+ assert result == [{"role": "assistant", "content": "The answer is 4."}]
+
+
def test_convert_assistant_message_tool_use():
content = [
MockBlock(type="text", text="I will call the tool."),
@@ -497,7 +557,7 @@ def test_assistant_redacted_thinking_omitted_from_openai_chat():
]
messages = [MockMessage("assistant", content)]
result = AnthropicToOpenAIConverter.convert_messages(
- messages, include_thinking=True, include_reasoning_content=True
+ messages, reasoning_replay=ReasoningReplayMode.REASONING_CONTENT
)
assert result[0]["content"] == "Visible."
assert "secret-opaque" not in result[0]["content"]
diff --git a/tests/providers/test_deepseek.py b/tests/providers/test_deepseek.py
index 7a4d9976..0c82fbf1 100644
--- a/tests/providers/test_deepseek.py
+++ b/tests/providers/test_deepseek.py
@@ -11,9 +11,10 @@ from providers.exceptions import InvalidRequestError
class MockMessage:
- def __init__(self, role, content):
+ def __init__(self, role, content, reasoning_content=None):
self.role = role
self.content = content
+ self.reasoning_content = reasoning_content
class MockBlock:
@@ -151,7 +152,61 @@ def test_build_request_body_preserves_reasoning_content(deepseek_provider):
body = deepseek_provider._build_request_body(req)
- assert body["messages"][0]["reasoning_content"] == "First think"
+ assistant = body["messages"][0]
+ assert assistant["reasoning_content"] == "First think"
+ assert "<think>" not in assistant["content"]
+ assert assistant["content"] == "Then answer"
+
+
+def test_build_request_body_preserves_top_level_reasoning_content(deepseek_provider):
+ """OpenAI-compatible assistant reasoning_content survives history replay."""
+ req = MockRequest(
+ system=None,
+ messages=[
+ MockMessage(
+ "assistant",
+ "Then answer",
+ reasoning_content="First think",
+ )
+ ],
+ )
+
+ body = deepseek_provider._build_request_body(req)
+
+ assistant = body["messages"][0]
+ assert assistant["reasoning_content"] == "First think"
+ assert assistant["content"] == "Then answer"
+
+
+def test_build_request_body_replays_reasoning_content_with_tool_calls(
+ deepseek_provider,
+):
+ """Thinking plus tool_use history is replayed in DeepSeek's top-level shape."""
+ req = MockRequest(
+ system=None,
+ messages=[
+ MockMessage(
+ "assistant",
+ [
+ MockBlock(type="thinking", thinking="Need the tool."),
+ MockBlock(
+ type="tool_use",
+ id="call_reasoning",
+ name="echo",
+ input={"value": "x"},
+ ),
+ ],
+ )
+ ],
+ )
+
+ body = deepseek_provider._build_request_body(req)
+
+ assistant = body["messages"][0]
+ assert assistant["content"] == ""
+ assert assistant["reasoning_content"] == "Need the tool."
+ assert assistant["tool_calls"][0]["id"] == "call_reasoning"
+ assert "<think>" not in assistant["content"]
def test_build_request_body_disabled_thinking_omits_reasoning_and_thinking_tags():
diff --git a/tests/providers/test_nvidia_nim.py b/tests/providers/test_nvidia_nim.py
index 4c190ed4..f9a9095d 100644
--- a/tests/providers/test_nvidia_nim.py
+++ b/tests/providers/test_nvidia_nim.py
@@ -24,6 +24,12 @@ class MockTool:
self.input_schema = input_schema
+class MockBlock:
+ def __init__(self, **kwargs):
+ for key, value in kwargs.items():
+ setattr(self, key, value)
+
+
class MockRequest:
def __init__(self, **kwargs):
self.model = "test-model"
@@ -429,6 +435,57 @@ async def test_stream_response_retries_without_reasoning_budget(nim_provider):
assert any("message_stop" in event for event in events)
+async def test_stream_response_retries_without_reasoning_content(nim_provider):
+ req = MockRequest(
+ system=None,
+ messages=[
+ MockMessage(
+ "assistant",
+ [
+ MockBlock(type="thinking", thinking="Need the tool."),
+ MockBlock(
+ type="tool_use",
+ id="toolu_reasoning",
+ name="echo_smoke",
+ input={"value": "FCC_TOOL"},
+ ),
+ ],
+ )
+ ],
+ )
+
+ mock_chunk = MagicMock()
+ mock_chunk.choices = [
+ MagicMock(
+ delta=MagicMock(content="Recovered", reasoning_content=""),
+ finish_reason="stop",
+ )
+ ]
+ mock_chunk.usage = MagicMock(completion_tokens=5)
+
+ async def mock_stream():
+ yield mock_chunk
+
+ error = _make_bad_request_error("Unsupported field: reasoning_content")
+
+ with patch.object(
+ nim_provider._client.chat.completions, "create", new_callable=AsyncMock
+ ) as mock_create:
+ mock_create.side_effect = [error, mock_stream()]
+
+ events = [e async for e in nim_provider.stream_response(req)]
+
+ assert mock_create.await_count == 2
+ first_call = mock_create.await_args_list[0].kwargs
+ second_call = mock_create.await_args_list[1].kwargs
+ assert first_call["messages"][0]["reasoning_content"] == "Need the tool."
+ assert "reasoning_content" not in second_call["messages"][0]
+ assert second_call["messages"][0]["tool_calls"][0]["id"] == "toolu_reasoning"
+ assert any("Recovered" in event for event in events)
+ assert any("message_stop" in event for event in events)
+
+
@pytest.mark.asyncio
async def test_stream_response_bad_request_without_reasoning_budget_does_not_retry(
nim_provider,
diff --git a/tests/providers/test_nvidia_nim_request.py b/tests/providers/test_nvidia_nim_request.py
index df38e245..e3137a35 100644
--- a/tests/providers/test_nvidia_nim_request.py
+++ b/tests/providers/test_nvidia_nim_request.py
@@ -10,6 +10,7 @@ from providers.nvidia_nim.request import (
_set_extra,
build_request_body,
clone_body_without_chat_template,
+ clone_body_without_reasoning_content,
)
@@ -253,3 +254,56 @@ class TestBuildRequestBody:
body = build_request_body(req, NimSettings(), thinking_enabled=False)
assert "<think>" not in body["messages"][0]["content"]
assert "answer" in body["messages"][0]["content"]
+
+ def test_assistant_thinking_replayed_as_reasoning_content_when_enabled(self):
+ req = MagicMock()
+ req.model = "test"
+ req.messages = [
+ MagicMock(
+ role="assistant",
+ content=[
+ MagicMock(type="thinking", thinking="secret"),
+ MagicMock(type="text", text="answer"),
+ ],
+ reasoning_content=None,
+ )
+ ]
+ req.max_tokens = 100
+ req.system = None
+ req.temperature = None
+ req.top_p = None
+ req.stop_sequences = None
+ req.tools = None
+ req.tool_choice = None
+ req.extra_body = None
+ req.top_k = None
+
+ body = build_request_body(req, NimSettings(), thinking_enabled=True)
+ assistant = body["messages"][0]
+ assert assistant["reasoning_content"] == "secret"
+ assert assistant["content"] == "answer"
+ assert "<think>" not in assistant["content"]
+
+ def test_clone_body_without_reasoning_content(self):
+ body = {
+ "model": "test",
+ "messages": [
+ {"role": "user", "content": "hi"},
+ {
+ "role": "assistant",
+ "content": "answer",
+ "reasoning_content": "secret",
+ },
+ ],
+ }
+
+ cloned = clone_body_without_reasoning_content(body)
+
+ assert cloned is not None
+ assert "reasoning_content" not in cloned["messages"][1]
+ assert body["messages"][1]["reasoning_content"] == "secret"
+
+ def test_clone_body_without_reasoning_content_returns_none_when_unchanged(self):
+ body = {"model": "test", "messages": [{"role": "user", "content": "hi"}]}
+
+ assert clone_body_without_reasoning_content(body) is None