Skip to content

Use Claude Code, Codex, VSCode, Pi, and OpenCode (and 6 other harnesses) for free (1.3B+ free tokens) from your terminal, app, IDE, or phone, and now from the browser with native browser sessions (multi-harness + multi-model) like OpenClaw (voice supported + ToS friendly)

Python

Improve smoke provider coverage and skips

Alishahryar1b867e08084dc
diff --git a/pyproject.toml b/pyproject.toml
index 06ce0d51..1c32b357 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -118,6 +118,7 @@ markers = [
"voice: voice transcription checks",
"contract: deterministic feature contract checks",
"smoke_target(name): route a smoke test behind FCC_SMOKE_TARGETS",
+ "xdist_group(name): group smoke provider items under pytest-xdist --dist=loadgroup",
]
[tool.ty.environment]
diff --git a/smoke/README.md b/smoke/README.md
index 414a43d9..871f4159 100644
--- a/smoke/README.md
+++ b/smoke/README.md
@@ -31,6 +31,15 @@ $env:FCC_LIVE_SMOKE = "1"
uv run pytest smoke -n 0 -s --tb=short
```
+Provider smoke scenarios can run providers in parallel while preserving
+sequential execution within each provider:
+
+```powershell
+$env:FCC_LIVE_SMOKE = "1"
+$env:FCC_SMOKE_TARGETS = "providers"
+uv run pytest smoke -n auto --dist=loadgroup -s --tb=short
+```
+
Provider product E2E runs once per configured provider, independent of `MODEL`,
`MODEL_OPUS`, `MODEL_SONNET`, and `MODEL_HAIKU`. Defaults come from the provider
catalog/docs and can be overridden with `FCC_SMOKE_MODEL_<PROVIDER>`, for example
@@ -100,7 +109,7 @@ uv run pytest smoke/product -n 0 -s --tb=short
```powershell
$env:FCC_LIVE_SMOKE = "1"
$env:FCC_SMOKE_TARGETS = "openrouter_free_cli"
-$env:FCC_SMOKE_OPENROUTER_FREE_MODELS = "nvidia/nemotron-3-super-120b-a12b:free,openai/gpt-oss-120b:free,minimax/minimax-m2.5:free,inclusionai/ring-2.6-1t:free,poolside/laguna-m.1:free"
+$env:FCC_SMOKE_OPENROUTER_FREE_MODELS = "nvidia/nemotron-3-super-120b-a12b:free,openai/gpt-oss-120b:free,poolside/laguna-m.1:free"
uv run pytest smoke/product -n 0 -s --tb=short
```
@@ -154,10 +163,9 @@ locked.
Smoke artifacts are written to `.smoke-results/` and redact env values whose
names contain `KEY`, `TOKEN`, `SECRET`, `WEBHOOK`, or `AUTH`.
-- `missing_env`: required credentials, binary, provider config, local server, or
- opt-in flag is absent.
-- `upstream_unavailable`: a real provider, bot API, or local model server is not
- reachable.
+- `missing_env`: required credentials, binary, provider config, local provider
+ server/model, or opt-in flag is absent.
+- `upstream_unavailable`: a real provider or bot API is not reachable.
- `probe_timeout`: the smoke driver reached the target, but the CLI/probe did
not complete within the smoke timeout.
- `product_failure`: the app accepted the scenario but returned the wrong shape,
diff --git a/smoke/conftest.py b/smoke/conftest.py
index a10e413e..264324aa 100644
--- a/smoke/conftest.py
+++ b/smoke/conftest.py
@@ -1,13 +1,26 @@
from __future__ import annotations
from collections.abc import Iterator
+from typing import Any
import pytest
-from smoke.lib.config import SmokeConfig, auth_headers
+from smoke.lib.config import ProviderModel, SmokeConfig, auth_headers
from smoke.lib.report import SmokeReport
from smoke.lib.server import RunningServer, start_server
+DISABLED_PROVIDER_MODEL = ProviderModel(
+ provider="smoke_disabled",
+ full_model="smoke_disabled/smoke-disabled",
+ source="smoke_disabled",
+)
+
+
+def pytest_generate_tests(metafunc: pytest.Metafunc) -> None:
+ if "provider_model" in metafunc.fixturenames:
+ config = SmokeConfig.load()
+ metafunc.parametrize("provider_model", provider_model_params(config))
+
def pytest_collection_modifyitems(items: list[pytest.Item]) -> None:
if SmokeConfig.load().live:
@@ -74,4 +87,42 @@ def smoke_headers() -> dict[str, str]:
return auth_headers()
+def provider_model_params(config: SmokeConfig) -> list[Any]:
+ """Return provider params grouped for pytest-xdist ``--dist=loadgroup``."""
+ if not config.live:
+ return [_disabled_provider_param("set FCC_LIVE_SMOKE=1 to run provider smoke")]
+
+ models = config.provider_smoke_models()
+ if not models:
+ return [_disabled_provider_param("missing_env: no configured provider smoke")]
+
+ return [
+ pytest.param(
+ model,
+ id=provider_model_id(model),
+ marks=pytest.mark.xdist_group(provider_xdist_group(model)),
+ )
+ for model in models
+ ]
+
+
+def _disabled_provider_param(reason: str) -> Any:
+ return pytest.param(
+ DISABLED_PROVIDER_MODEL,
+ id=provider_model_id(DISABLED_PROVIDER_MODEL),
+ marks=(
+ pytest.mark.skip(reason=reason),
+ pytest.mark.xdist_group(provider_xdist_group(DISABLED_PROVIDER_MODEL)),
+ ),
+ )
+
+
+def provider_model_id(provider_model: ProviderModel) -> str:
+ return provider_model.provider
+
+
+def provider_xdist_group(provider_model: ProviderModel) -> str:
+ return f"provider:{provider_model.provider}"
+
+
_REPORT: SmokeReport | None = None
diff --git a/smoke/lib/config.py b/smoke/lib/config.py
index f13abc5a..0b58fddb 100644
--- a/smoke/lib/config.py
+++ b/smoke/lib/config.py
@@ -43,7 +43,7 @@ SECRET_KEY_PARTS = ("KEY", "TOKEN", "SECRET", "WEBHOOK", "AUTH")
PROVIDER_SMOKE_DEFAULT_MODELS: dict[str, str] = {
"nvidia_nim": "nvidia_nim/nvidia/nemotron-3-super-120b-a12b",
- "open_router": "open_router/openrouter/free",
+ "open_router": "open_router/moonshotai/kimi-k2.6:free",
"mistral": "mistral/devstral-small-latest",
"mistral_codestral": "mistral_codestral/codestral-latest",
"deepseek": "deepseek/deepseek-v4-pro",
@@ -71,8 +71,6 @@ NVIDIA_NIM_CLI_DEFAULT_MODELS: tuple[str, ...] = (
OPENROUTER_FREE_CLI_DEFAULT_MODELS: tuple[str, ...] = (
"nvidia/nemotron-3-super-120b-a12b:free",
"openai/gpt-oss-120b:free",
- "minimax/minimax-m2.5:free",
- "inclusionai/ring-2.6-1t:free",
"poolside/laguna-m.1:free",
)
diff --git a/smoke/lib/local_providers.py b/smoke/lib/local_providers.py
new file mode 100644
index 00000000..ba2be5d3
--- /dev/null
+++ b/smoke/lib/local_providers.py
@@ -0,0 +1,85 @@
+"""Helpers for local-provider smoke availability checks."""
+
+from __future__ import annotations
+
+from urllib.parse import urljoin
+
+import httpx
+import pytest
+
+LOCAL_PROVIDER_PROBE_TIMEOUT_S = 1.5
+
+
+def first_local_provider_model_id(
+ provider: str,
+ base_url: str,
+ *,
+ timeout_s: float,
+) -> str:
+ """Return the first local model id, or skip when the local server is absent."""
+ base_url = base_url.strip()
+ if not base_url:
+ pytest.skip(f"missing_env: {provider} base URL is not configured")
+
+ timeout = min(timeout_s, LOCAL_PROVIDER_PROBE_TIMEOUT_S)
+ if provider == "ollama":
+ return _first_ollama_model_id(base_url, timeout_s=timeout)
+ return _first_openai_compatible_model_id(
+ provider,
+ base_url,
+ timeout_s=timeout,
+ )
+
+
+def _first_openai_compatible_model_id(
+ provider: str,
+ base_url: str,
+ *,
+ timeout_s: float,
+) -> str:
+ models_url = urljoin(base_url.rstrip("/") + "/", "models")
+ response = _get_local_provider_response(provider, models_url, timeout_s=timeout_s)
+ assert response.status_code == 200, response.text
+ payload = response.json()
+ data = payload.get("data") if isinstance(payload, dict) else None
+ if isinstance(data, list):
+ for item in data:
+ if isinstance(item, dict) and isinstance(item.get("id"), str):
+ return item["id"]
+ pytest.skip(f"missing_env: {provider} local server has no loaded models")
+ pytest.fail("product_failure: local /models did not expose a model id")
+
+
+def _first_ollama_model_id(base_url: str, *, timeout_s: float) -> str:
+ tags_url = f"{base_url.rstrip('/')}/api/tags"
+ response = _get_local_provider_response("ollama", tags_url, timeout_s=timeout_s)
+ assert response.status_code == 200, response.text
+ payload = response.json()
+ models = payload.get("models") if isinstance(payload, dict) else None
+ if isinstance(models, list):
+ for item in models:
+ if isinstance(item, dict) and isinstance(item.get("name"), str):
+ return item["name"]
+ pytest.skip("missing_env: ollama local server has no pulled models")
+ pytest.fail("product_failure: ollama /api/tags did not expose models")
+
+
+def _get_local_provider_response(
+ provider: str,
+ url: str,
+ *,
+ timeout_s: float,
+) -> httpx.Response:
+ try:
+ response = httpx.get(url, timeout=timeout_s)
+ except httpx.TimeoutException as exc:
+ pytest.skip(f"missing_env: {provider} local server is not reachable: {exc}")
+ except httpx.NetworkError as exc:
+ pytest.skip(f"missing_env: {provider} local server is not running: {exc}")
+
+ if response.status_code in {404, 405, 502, 503}:
+ pytest.skip(
+ f"missing_env: {provider} local server is not available at {url}: "
+ f"HTTP {response.status_code}"
+ )
+ return response
diff --git a/smoke/lib/report.py b/smoke/lib/report.py
index ea3b0b38..8f499c46 100644
--- a/smoke/lib/report.py
+++ b/smoke/lib/report.py
@@ -71,6 +71,8 @@ def classify_outcome(*, nodeid: str, outcome: str, detail: str) -> str:
if outcome == "skipped":
if "smoke target disabled" in text:
return "target_disabled"
+ if "missing_env" in text:
+ return "missing_env"
if any(
marker in text
for marker in (
diff --git a/smoke/lib/skips.py b/smoke/lib/skips.py
index 1cb1e29a..73d66426 100644
--- a/smoke/lib/skips.py
+++ b/smoke/lib/skips.py
@@ -5,7 +5,7 @@ from __future__ import annotations
import httpx
import pytest
-from core.anthropic.stream_contracts import SSEEvent
+from core.anthropic.stream_contracts import SSEEvent, text_content
UPSTREAM_UNAVAILABLE_MARKERS = (
"connection refused",
@@ -50,6 +50,10 @@ def skip_if_upstream_unavailable_exception(exc: Exception) -> None:
def skip_if_upstream_unavailable_events(events: list[SSEEvent]) -> None:
+ text = text_content(events)
+ if is_upstream_unavailable_text(text):
+ skip_upstream_unavailable(text[:500])
+
for event in events:
if getattr(event, "event", None) != "error":
continue
diff --git a/smoke/prereq/test_local_provider_endpoints_prereq_live.py b/smoke/prereq/test_local_provider_endpoints_prereq_live.py
index 43f59c22..affedf9b 100644
--- a/smoke/prereq/test_local_provider_endpoints_prereq_live.py
+++ b/smoke/prereq/test_local_provider_endpoints_prereq_live.py
@@ -1,86 +1,36 @@
from __future__ import annotations
-import httpx
import pytest
from smoke.lib.config import SmokeConfig
-from smoke.lib.skips import skip_if_upstream_unavailable_exception
+from smoke.lib.local_providers import first_local_provider_model_id
@pytest.mark.live
@pytest.mark.smoke_target("lmstudio")
def test_lmstudio_models_endpoint_when_available(smoke_config: SmokeConfig) -> None:
- _assert_models_endpoint(
+ first_local_provider_model_id(
+ "lmstudio",
smoke_config.settings.lm_studio_base_url,
timeout_s=smoke_config.timeout_s,
- provider_name="LM Studio",
)
@pytest.mark.live
@pytest.mark.smoke_target("llamacpp")
def test_llamacpp_models_endpoint_when_available(smoke_config: SmokeConfig) -> None:
- _assert_models_endpoint(
+ first_local_provider_model_id(
+ "llamacpp",
smoke_config.settings.llamacpp_base_url,
timeout_s=smoke_config.timeout_s,
- provider_name="llama.cpp",
)
@pytest.mark.live
@pytest.mark.smoke_target("ollama")
def test_ollama_models_endpoint_when_available(smoke_config: SmokeConfig) -> None:
- _assert_ollama_tags_endpoint(
- smoke_config.settings.ollama_base_url, timeout_s=smoke_config.timeout_s
+ first_local_provider_model_id(
+ "ollama",
+ smoke_config.settings.ollama_base_url,
+ timeout_s=smoke_config.timeout_s,
)
-
-
-def _assert_models_endpoint(
- base_url: str, *, timeout_s: float, provider_name: str
-) -> None:
- url = f"{base_url.rstrip('/')}/models"
- try:
- response = httpx.get(url, timeout=timeout_s)
- except Exception as exc:
- skip_if_upstream_unavailable_exception(exc)
- raise
-
- if response.status_code in {404, 405, 502, 503}:
- pytest.skip(
- f"upstream_unavailable: {provider_name} models endpoint "
- f"{url} returned HTTP {response.status_code}"
- )
-
- assert response.status_code == 200, response.text
- payload = response.json()
- data = payload.get("data")
- if isinstance(data, list) and data:
- return
- if isinstance(data, list):
- pytest.skip(f"upstream_unavailable: {provider_name} has no local models")
- assert isinstance(data, list), payload
-
-
-def _assert_ollama_tags_endpoint(base_url: str, *, timeout_s: float) -> None:
- url = f"{_ollama_root_url(base_url)}/api/tags"
- try:
- response = httpx.get(url, timeout=timeout_s)
- except Exception as exc:
- skip_if_upstream_unavailable_exception(exc)
- raise
-
- if response.status_code in {404, 405, 502, 503}:
- pytest.skip(
- f"upstream_unavailable: Ollama tags endpoint {url} "
- f"returned HTTP {response.status_code}"
- )
-
- assert response.status_code == 200, response.text
- models = response.json().get("models")
- if isinstance(models, list) and models:
- return
- pytest.skip("upstream_unavailable: Ollama has no pulled models")
-
-
-def _ollama_root_url(base_url: str) -> str:
- return base_url.rstrip("/")
diff --git a/smoke/prereq/test_provider_prereq_live.py b/smoke/prereq/test_provider_prereq_live.py
index c5269f10..b08e808d 100644
--- a/smoke/prereq/test_provider_prereq_live.py
+++ b/smoke/prereq/test_provider_prereq_live.py
@@ -10,7 +10,7 @@ from core.anthropic.stream_contracts import (
text_content,
thinking_content,
)
-from smoke.lib.config import SmokeConfig, auth_headers
+from smoke.lib.config import ProviderModel, SmokeConfig, auth_headers
from smoke.lib.e2e import ProviderMatrixDriver
from smoke.lib.http import collect_message_stream, message_payload
from smoke.lib.server import start_server
@@ -45,41 +45,35 @@ def test_mixed_provider_model_mapping_when_configured(
def test_configured_provider_models_stream_successfully(
- smoke_config: SmokeConfig,
+ smoke_config: SmokeConfig, provider_model: ProviderModel
) -> None:
- models = ProviderMatrixDriver(smoke_config).provider_smoke_models()
-
- failures: list[str] = []
- for provider_model in models:
- try:
- with start_server(
+ try:
+ with start_server(
+ smoke_config,
+ env_overrides={
+ "MODEL": provider_model.full_model,
+ "MESSAGING_PLATFORM": "none",
+ },
+ name=f"provider-{provider_model.provider}",
+ ) as server:
+ events = collect_message_stream(
+ server,
+ message_payload(smoke_config.prompt, model="fcc-smoke-default"),
smoke_config,
- env_overrides={
- "MODEL": provider_model.full_model,
- "MESSAGING_PLATFORM": "none",
- },
- name=f"provider-{provider_model.provider}",
- ) as server:
- events = collect_message_stream(
- server,
- message_payload(smoke_config.prompt, model="fcc-smoke-default"),
- smoke_config,
- )
- skip_if_upstream_unavailable_events(events)
- assert_anthropic_stream_contract(events)
- has_text = bool(text_content(events).strip())
- has_thinking = bool(thinking_content(events).strip())
- assert has_text or has_thinking, (
- "provider returned no visible text or thinking content"
- )
- except Exception as exc:
- skip_if_upstream_unavailable_exception(exc)
- failures.append(
- f"{provider_model.source}={provider_model.full_model}: "
- f"{type(exc).__name__}: {exc}"
)
-
- assert not failures, "\n".join(failures)
+ skip_if_upstream_unavailable_events(events)
+ assert_anthropic_stream_contract(events)
+ has_text = bool(text_content(events).strip())
+ has_thinking = bool(thinking_content(events).strip())
+ assert has_text or has_thinking, (
+ "provider returned no visible text or thinking content"
+ )
+ except Exception as exc:
+ skip_if_upstream_unavailable_exception(exc)
+ raise AssertionError(
+ f"{provider_model.source}={provider_model.full_model}: "
+ f"{type(exc).__name__}: {exc}"
+ ) from exc
@pytest.mark.smoke_target("rate_limit")
diff --git a/smoke/product/test_local_provider_product_live.py b/smoke/product/test_local_provider_product_live.py
index a8fa519e..5d3797ce 100644
--- a/smoke/product/test_local_provider_product_live.py
+++ b/smoke/product/test_local_provider_product_live.py
@@ -1,12 +1,10 @@
from __future__ import annotations
-from urllib.parse import urljoin
-
-import httpx
import pytest
from smoke.lib.config import SmokeConfig
from smoke.lib.e2e import ConversationDriver, SmokeServerDriver, assert_product_stream
+from smoke.lib.local_providers import first_local_provider_model_id
pytestmark = [pytest.mark.live]
@@ -44,13 +42,10 @@ def _local_native_messages_e2e(
provider: str,
base_url: str,
) -> None:
- if not base_url.strip():
- pytest.skip(f"missing_env: {provider} base URL is not configured")
-
- model_id = (
- _first_ollama_tag_model_id(base_url)
- if provider == "ollama"
- else (_first_non_ollama_model_id(provider, base_url))
+ model_id = first_local_provider_model_id(
+ provider,
+ base_url,
+ timeout_s=smoke_config.timeout_s,
)
with SmokeServerDriver(
@@ -63,46 +58,3 @@ def _local_native_messages_e2e(
)
assert_product_stream(turn.events)
-
-
-def _first_non_ollama_model_id(provider: str, base_url: str) -> str:
- models_url = urljoin(base_url.rstrip("/") + "/", "models")
- try:
- response = httpx.get(models_url, timeout=5)
- except httpx.ConnectError as exc:
- pytest.skip(f"upstream_unavailable: {provider} models endpoint: {exc}")
- except httpx.TimeoutException as exc:
- pytest.skip(f"upstream_unavailable: {provider} models endpoint: {exc}")
- assert response.status_code == 200, response.text
- payload = response.json()
- data = payload.get("data") if isinstance(payload, dict) else None
- if isinstance(data, list):
- for item in data:
- if isinstance(item, dict) and isinstance(item.get("id"), str):
- return item["id"]
- pytest.skip(f"upstream_unavailable: {provider} has no local models")
- pytest.fail("product_failure: local /models did not expose a model id")
-
-
-def _first_ollama_tag_model_id(base_url: str) -> str:
- tags_url = f"{_ollama_root_url(base_url)}/api/tags"
- try:
- response = httpx.get(tags_url, timeout=5)
- except httpx.ConnectError as exc:
- pytest.skip(f"upstream_unavailable: ollama tags endpoint: {exc}")
- except httpx.TimeoutException as exc:
- pytest.skip(f"upstream_unavailable: ollama tags endpoint: {exc}")
-
- assert response.status_code == 200, response.text
- payload = response.json()
- models = payload.get("models") if isinstance(payload, dict) else None
- if isinstance(models, list):
- for item in models:
- if isinstance(item, dict) and isinstance(item.get("name"), str):
- return item["name"]
- pytest.skip("upstream_unavailable: ollama has no pulled models")
- pytest.fail("product_failure: ollama /api/tags did not expose models")
-
-
-def _ollama_root_url(base_url: str) -> str:
- return base_url.rstrip("/")
diff --git a/smoke/product/test_provider_product_live.py b/smoke/product/test_provider_product_live.py
index 593d6412..6bcee620 100644
--- a/smoke/product/test_provider_product_live.py
+++ b/smoke/product/test_provider_product_live.py
@@ -7,6 +7,7 @@ import pytest
from config.provider_catalog import PROVIDER_CATALOG
from core.anthropic.stream_contracts import (
+ SSEEvent,
assert_anthropic_stream_contract,
parse_sse_lines,
text_content,
@@ -42,51 +43,74 @@ def test_model_mapping_matrix_e2e(smoke_config: SmokeConfig) -> None:
assert model.model_name
-def test_provider_text_multiturn_e2e(smoke_config: SmokeConfig) -> None:
- _run_for_each_provider(smoke_config, _scenario_text_multiturn)
+def test_provider_text_multiturn_e2e(
+ smoke_config: SmokeConfig, provider_model: ProviderModel
+) -> None:
+ _run_provider_scenario(smoke_config, provider_model, _scenario_text_multiturn)
-def test_provider_adaptive_thinking_history_e2e(smoke_config: SmokeConfig) -> None:
- _run_for_each_provider(smoke_config, _scenario_adaptive_thinking_history)
+def test_provider_adaptive_thinking_history_e2e(
+ smoke_config: SmokeConfig, provider_model: ProviderModel
+) -> None:
+ _run_provider_scenario(
+ smoke_config, provider_model, _scenario_adaptive_thinking_history
+ )
-def test_provider_interleaved_thinking_tool_e2e(smoke_config: SmokeConfig) -> None:
- _run_for_each_provider(smoke_config, _scenario_interleaved_history)
+def test_provider_interleaved_thinking_tool_e2e(
+ smoke_config: SmokeConfig, provider_model: ProviderModel
+) -> None:
+ _run_provider_scenario(smoke_config, provider_model, _scenario_interleaved_history)
@pytest.mark.smoke_target("tools")
-def test_provider_tool_use_then_text_history_e2e(smoke_config: SmokeConfig) -> None:
+def test_provider_tool_use_then_text_history_e2e(
+ smoke_config: SmokeConfig, provider_model: ProviderModel
+) -> None:
"""OpenAI-compatible path: history with tool_use + assistant text after tool (issue #206)."""
- _run_for_each_provider(smoke_config, _scenario_tool_use_then_text_in_history)
+ _run_provider_scenario(
+ smoke_config, provider_model, _scenario_tool_use_then_text_in_history
+ )
@pytest.mark.smoke_target("tools")
-def test_provider_tool_result_continuation_e2e(smoke_config: SmokeConfig) -> None:
- _run_for_each_provider(smoke_config, _scenario_tool_result_continuation)
+def test_provider_tool_result_continuation_e2e(
+ smoke_config: SmokeConfig, provider_model: ProviderModel
+) -> None:
+ _run_provider_scenario(
+ smoke_config, provider_model, _scenario_tool_result_continuation
+ )
@pytest.mark.smoke_target("tools")
def test_gemini_thought_signature_tool_continuation_e2e(
- smoke_config: SmokeConfig,
+ smoke_config: SmokeConfig, provider_model: ProviderModel
) -> None:
- models = [
- model
- for model in ProviderMatrixDriver(smoke_config).provider_smoke_models()
- if model.provider == "gemini"
- ]
- if not models:
- pytest.skip("missing_env: gemini is not configured")
- _scenario_gemini_thought_signature_tool_continuation(smoke_config, models[0])
+ if provider_model.provider != "gemini":
+ pytest.skip("gemini-specific smoke scenario")
+ _run_provider_scenario(
+ smoke_config,
+ provider_model,
+ _scenario_gemini_thought_signature_tool_continuation,
+ )
@pytest.mark.smoke_target("tools")
-def test_provider_reasoning_tool_continuation_e2e(smoke_config: SmokeConfig) -> None:
- _run_for_each_thinking_provider(smoke_config, _scenario_reasoning_tool_continuation)
+def test_provider_reasoning_tool_continuation_e2e(
+ smoke_config: SmokeConfig, provider_model: ProviderModel
+) -> None:
+ if not _provider_smoke_thinking_enabled(smoke_config, provider_model):
+ pytest.skip(f"{provider_model.provider} smoke model does not enable thinking")
+ _run_provider_scenario(
+ smoke_config, provider_model, _scenario_reasoning_tool_continuation
+ )
@pytest.mark.smoke_target("rate_limit")
-def test_provider_disconnect_e2e(smoke_config: SmokeConfig) -> None:
- _run_for_each_provider(smoke_config, _scenario_disconnect)
+def test_provider_disconnect_e2e(
+ smoke_config: SmokeConfig, provider_model: ProviderModel
+) -> None:
+ _run_provider_scenario(smoke_config, provider_model, _scenario_disconnect)
def test_provider_error_e2e(smoke_config: SmokeConfig) -> None:
@@ -147,42 +171,36 @@ def test_openrouter_native_e2e(smoke_config: SmokeConfig) -> None:
"thinking": {"type": "adaptive", "budget_tokens": 1024},
}
)
- assert_product_stream(turn.events)
+ _assert_provider_product_stream(turn.events)
-def _run_for_each_provider(smoke_config: SmokeConfig, scenario) -> None:
- failures: list[str] = []
- for provider_model in ProviderMatrixDriver(smoke_config).provider_smoke_models():
- try:
- scenario(smoke_config, provider_model)
- except Exception as exc:
- skip_if_upstream_unavailable_exception(exc)
- failures.append(
- f"{provider_model.source}={provider_model.full_model}: "
- f"{type(exc).__name__}: {exc}"
- )
- assert not failures, "\n".join(failures)
+def _run_provider_scenario(
+ smoke_config: SmokeConfig,
+ provider_model: ProviderModel,
+ scenario,
+) -> None:
+ try:
+ scenario(smoke_config, provider_model)
+ except Exception as exc:
+ skip_if_upstream_unavailable_exception(exc)
+ raise AssertionError(
+ f"{provider_model.source}={provider_model.full_model}: "
+ f"{type(exc).__name__}: {exc}"
+ ) from exc
-def _run_for_each_thinking_provider(smoke_config: SmokeConfig, scenario) -> None:
- failures: list[str] = []
- models = [
- provider_model
- for provider_model in ProviderMatrixDriver(smoke_config).provider_smoke_models()
- if _provider_smoke_thinking_enabled(smoke_config, provider_model)
- ]
- if not models:
- pytest.skip("missing_env: no thinking-capable provider smoke model configured")
- for provider_model in models:
- try:
- scenario(smoke_config, provider_model)
- except Exception as exc:
- skip_if_upstream_unavailable_exception(exc)
- failures.append(
- f"{provider_model.source}={provider_model.full_model}: "
- f"{type(exc).__name__}: {exc}"
- )
- assert not failures, "\n".join(failures)
+def _assert_provider_product_stream(events: list[SSEEvent]) -> None:
+ skip_if_upstream_unavailable_events(events)
+ assert_product_stream(events)
+
+
+def _tool_use_blocks_or_skip(
+ events: list[SSEEvent], message: str
+) -> list[dict[str, Any]]:
+ skip_if_upstream_unavailable_events(events)
+ blocks = tool_use_blocks(events)
+ assert blocks, message
+ return blocks
def _provider_smoke_thinking_enabled(
@@ -202,8 +220,8 @@ def _scenario_text_multiturn(
driver = ConversationDriver(server, smoke_config)
first = driver.ask("Reply with one short sentence.")
second = driver.ask("Reply with a different short sentence.")
- assert_product_stream(first.events)
- assert_product_stream(second.events)
+ _assert_provider_product_stream(first.events)
+ _assert_provider_product_stream(second.events)
def _scenario_adaptive_thinking_history(
@@ -228,7 +246,7 @@ def _scenario_adaptive_thinking_history(
}
with _server_for_provider(smoke_config, provider_model, "adaptive") as server:
turn = ConversationDriver(server, smoke_config).stream(payload)
- assert_product_stream(turn.events)
+ _assert_provider_product_stream(turn.events)
def _scenario_interleaved_history(
@@ -268,7 +286,7 @@ def _scenario_interleaved_history(
}
with _server_for_provider(smoke_config, provider_model, "interleaved") as server:
turn = ConversationDriver(server, smoke_config).stream(payload)
- assert_product_stream(turn.events)
+ _assert_provider_product_stream(turn.events)
def _scenario_tool_use_then_text_in_history(
@@ -314,7 +332,7 @@ def _scenario_tool_use_then_text_in_history(
}
with _server_for_provider(smoke_config, provider_model, "tool-206") as server:
turn = ConversationDriver(server, smoke_config).stream(payload)
- assert_product_stream(turn.events)
+ _assert_provider_product_stream(turn.events)
def _scenario_tool_result_continuation(
@@ -333,8 +351,9 @@ def _scenario_tool_result_continuation(
with _server_for_provider(smoke_config, provider_model, "tool") as server:
driver = ConversationDriver(server, smoke_config)
first = driver.stream(first_payload)
- tool_uses = tool_use_blocks(first.events)
- assert tool_uses, "provider did not emit a tool_use block"
+ tool_uses = _tool_use_blocks_or_skip(
+ first.events, "provider did not emit a tool_use block"
+ )
tool_use = tool_uses[0]
second_payload = {
"model": "claude-sonnet-4-5-20250929",
@@ -356,8 +375,8 @@ def _scenario_tool_result_continuation(
"tools": [echo_tool_schema()],
}
second = driver.stream(second_payload)
- assert_product_stream(first.events)
- assert_product_stream(second.events)
+ _assert_provider_product_stream(first.events)
+ _assert_provider_product_stream(second.events)
def _scenario_gemini_thought_signature_tool_continuation(
@@ -378,8 +397,9 @@ def _scenario_gemini_thought_signature_tool_continuation(
) as server:
driver = ConversationDriver(server, smoke_config)
first = driver.stream(first_payload)
- tool_uses = tool_use_blocks(first.events)
- assert tool_uses, "gemini did not emit a tool_use block"
+ tool_uses = _tool_use_blocks_or_skip(
+ first.events, "gemini did not emit a tool_use block"
+ )
tool_use = tool_uses[0]
signature = _gemini_tool_thought_signature(tool_use)
assert signature, (
@@ -406,8 +426,8 @@ def _scenario_gemini_thought_signature_tool_continuation(
"thinking": {"type": "adaptive", "budget_tokens": 1024},
}
second = driver.stream(second_payload)
- assert_product_stream(first.events)
- assert_product_stream(second.events)
+ _assert_provider_product_stream(first.events)
+ _assert_provider_product_stream(second.events)
def _gemini_tool_thought_signature(tool_use: dict[str, Any]) -> str | None:
@@ -457,7 +477,7 @@ def _scenario_reasoning_tool_continuation(
}
with _server_for_provider(smoke_config, provider_model, "reasoning-tool") as server:
turn = ConversationDriver(server, smoke_config).stream(payload)
- assert_product_stream(turn.events)
+ _assert_provider_product_stream(turn.events)
def _scenario_disconnect(
@@ -483,8 +503,7 @@ def _scenario_disconnect(
followup = ConversationDriver(server, smoke_config).ask(
"Reply with one short sentence."
)
- skip_if_upstream_unavailable_events(followup.events)
- assert_product_stream(followup.events)
+ _assert_provider_product_stream(followup.events)
def _server_for_provider(
diff --git a/tests/contracts/test_local_provider_smoke.py b/tests/contracts/test_local_provider_smoke.py
new file mode 100644
index 00000000..a5ea7b32
--- /dev/null
+++ b/tests/contracts/test_local_provider_smoke.py
@@ -0,0 +1,89 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+import httpx
+import pytest
+
+from smoke.lib.local_providers import first_local_provider_model_id
+
+
+@dataclass(slots=True)
+class FakeResponse:
+ status_code: int
+ payload: dict
+ text: str = ""
+
+ def json(self) -> dict:
+ return self.payload
+
+
+def test_local_provider_openai_models_returns_first_id(monkeypatch) -> None:
+ calls: list[tuple[str, float]] = []
+
+ def fake_get(url: str, *, timeout: float) -> FakeResponse:
+ calls.append((url, timeout))
+ return FakeResponse(200, {"data": [{"id": "local-model"}]})
+
+ monkeypatch.setattr("smoke.lib.local_providers.httpx.get", fake_get)
+
+ model = first_local_provider_model_id(
+ "lmstudio",
+ "http://127.0.0.1:1234/v1",
+ timeout_s=45,
+ )
+
+ assert model == "local-model"
+ assert calls == [("http://127.0.0.1:1234/v1/models", 1.5)]
+
+
+def test_local_provider_ollama_tags_returns_first_name(monkeypatch) -> None:
+ def fake_get(url: str, *, timeout: float) -> FakeResponse:
+ assert url == "http://127.0.0.1:11434/api/tags"
+ assert timeout == 1.5
+ return FakeResponse(200, {"models": [{"name": "llama3.1"}]})
+
+ monkeypatch.setattr("smoke.lib.local_providers.httpx.get", fake_get)
+
+ assert (
+ first_local_provider_model_id(
+ "ollama",
+ "http://127.0.0.1:11434",
+ timeout_s=45,
+ )
+ == "llama3.1"
+ )
+
+
+def test_local_provider_not_running_is_missing_env_skip(monkeypatch) -> None:
+ def fake_get(url: str, *, timeout: float) -> FakeResponse:
+ raise httpx.ConnectError("refused")
+
+ monkeypatch.setattr("smoke.lib.local_providers.httpx.get", fake_get)
+
+ with pytest.raises(pytest.skip.Exception) as excinfo:
+ first_local_provider_model_id(
+ "llamacpp",
+ "http://127.0.0.1:8080/v1",
+ timeout_s=45,
+ )
+
+ assert "missing_env: llamacpp local server is not running" in str(excinfo.value)
+
+
+def test_local_provider_empty_model_list_is_missing_env_skip(monkeypatch) -> None:
+ def fake_get(url: str, *, timeout: float) -> FakeResponse:
+ return FakeResponse(200, {"data": []})
+
+ monkeypatch.setattr("smoke.lib.local_providers.httpx.get", fake_get)
+
+ with pytest.raises(pytest.skip.Exception) as excinfo:
+ first_local_provider_model_id(
+ "lmstudio",
+ "http://127.0.0.1:1234/v1",
+ timeout_s=45,
+ )
+
+ assert "missing_env: lmstudio local server has no loaded models" in str(
+ excinfo.value
+ )
diff --git a/tests/contracts/test_nvidia_nim_cli_matrix.py b/tests/contracts/test_nvidia_nim_cli_matrix.py
index 4666de3e..ae45b4d6 100644
--- a/tests/contracts/test_nvidia_nim_cli_matrix.py
+++ b/tests/contracts/test_nvidia_nim_cli_matrix.py
@@ -342,8 +342,8 @@ def test_cli_matrix_structured_provider_error_is_upstream_unavailable(
duration_s=0.1,
)
outcome = make_outcome(
- model="minimax/minimax-m2.5:free",
- full_model="open_router/minimax/minimax-m2.5:free",
+ model="poolside/laguna-m.1:free",
+ full_model="open_router/poolside/laguna-m.1:free",
source="openrouter_free_cli_default",
feature="tool_use_roundtrip",
marker="FCC_OPENROUTER_FREE_TOOL",
diff --git a/tests/contracts/test_smoke_config.py b/tests/contracts/test_smoke_config.py
index 257eb2c2..c57aa8b0 100644
--- a/tests/contracts/test_smoke_config.py
+++ b/tests/contracts/test_smoke_config.py
@@ -3,6 +3,11 @@ from __future__ import annotations
from pathlib import Path
from types import SimpleNamespace
+from smoke.conftest import (
+ DISABLED_PROVIDER_MODEL,
+ provider_model_params,
+ provider_xdist_group,
+)
from smoke.lib.config import (
ALL_TARGETS,
DEFAULT_TARGETS,
@@ -110,6 +115,19 @@ def test_provider_smoke_models_cover_configured_providers_independent_of_model_m
assert models[0].source == "provider_default"
+def test_openrouter_provider_smoke_uses_concrete_free_model(monkeypatch) -> None:
+ monkeypatch.delenv("FCC_SMOKE_MODEL_OPEN_ROUTER", raising=False)
+ config = _smoke_config(
+ settings=_settings(open_router_api_key="openrouter-key", ollama_base_url="")
+ )
+
+ models = config.provider_smoke_models()
+
+ assert [model.provider for model in models] == ["open_router"]
+ assert models[0].full_model == "open_router/moonshotai/kimi-k2.6:free"
+ assert models[0].source == "provider_default"
+
+
def test_wafer_provider_configuration_uses_api_key(monkeypatch) -> None:
monkeypatch.delenv("FCC_SMOKE_MODEL_WAFER", raising=False)
config = _smoke_config(
@@ -202,6 +220,41 @@ def test_provider_smoke_matrix_filters_provider_catalog(monkeypatch) -> None:
]
+def test_provider_smoke_collection_params_are_grouped_by_provider(
+ monkeypatch,
+) -> None:
+ monkeypatch.delenv("FCC_SMOKE_MODEL_DEEPSEEK", raising=False)
+ monkeypatch.delenv("FCC_SMOKE_MODEL_NVIDIA_NIM", raising=False)
+ config = _smoke_config(
+ live=True,
+ settings=_settings(
+ deepseek_api_key="deepseek-key",
+ nvidia_nim_api_key="nim-key",
+ ollama_base_url="",
+ ),
+ )
+
+ params = provider_model_params(config)
+
+ assert [param.id for param in params] == ["nvidia_nim", "deepseek"]
+ groups = [
+ mark.args[0]
+ for param in params
+ for mark in param.marks
+ if mark.name == "xdist_group"
+ ]
+ assert groups == ["provider:nvidia_nim", "provider:deepseek"]
+
+
+def test_provider_smoke_collection_uses_disabled_placeholder_when_not_live() -> None:
+ config = _smoke_config(live=False, settings=_settings(ollama_base_url=""))
+
+ params = provider_model_params(config)
+
+ assert [param.values[0] for param in params] == [DISABLED_PROVIDER_MODEL]
+ assert provider_xdist_group(DISABLED_PROVIDER_MODEL) == "provider:smoke_disabled"
+
+
def test_provider_smoke_includes_local_provider_when_model_mapping_uses_it(
monkeypatch,
) -> None:
diff --git a/tests/contracts/test_smoke_tiers.py b/tests/contracts/test_smoke_tiers.py
index ee92416b..85a4855b 100644
--- a/tests/contracts/test_smoke_tiers.py
+++ b/tests/contracts/test_smoke_tiers.py
@@ -3,8 +3,12 @@ from __future__ import annotations
import json
from pathlib import Path
+import pytest
+
+from core.anthropic.stream_contracts import SSEEvent
from smoke.lib.report import classify_outcome
from smoke.lib.report_summary import format_summary, summarize_reports
+from smoke.lib.skips import skip_if_upstream_unavailable_events
def test_smoke_readme_uses_env_gated_serial_commands() -> None:
@@ -43,3 +47,44 @@ def test_target_disabled_skip_is_not_missing_env() -> None:
)
assert classification == "target_disabled"
+
+
+def test_explicit_missing_env_skip_wins_over_network_words() -> None:
+ classification = classify_outcome(
+ nodeid="smoke/prereq/test_local_provider_endpoints_prereq_live.py::"
+ "test_ollama_endpoint_prereq_live",
+ outcome="skipped",
+ detail="Skipped: missing_env: ollama local server is not reachable: timed out",
+ )
+
+ assert classification == "missing_env"
+
+
+def test_provider_error_text_stream_is_upstream_unavailable_skip() -> None:
+ events = [
+ SSEEvent("message_start", {"message": {"content": []}}, ""),
+ SSEEvent(
+ "content_block_start",
+ {"index": 0, "content_block": {"type": "text", "text": ""}},
+ "",
+ ),
+ SSEEvent(
+ "content_block_delta",
+ {
+ "index": 0,
+ "delta": {
+ "type": "text_delta",
+ "text": "Upstream provider OPENROUTER returned HTTP 429.",
+ },
+ },
+ "",
+ ),
+ SSEEvent("content_block_stop", {"index": 0}, ""),
+ SSEEvent("message_delta", {"delta": {"stop_reason": "end_turn"}}, ""),
+ SSEEvent("message_stop", {}, ""),
+ ]
+
+ with pytest.raises(pytest.skip.Exception) as excinfo:
+ skip_if_upstream_unavailable_events(events)
+
+ assert "upstream_unavailable" in str(excinfo.value)
diff --git a/uv.lock b/uv.lock
index 8b90a3a0..6dd20cf6 100644
--- a/uv.lock
+++ b/uv.lock
@@ -554,7 +554,7 @@ wheels = [
[[package]]
name = "free-claude-code"
-version = "2.0.0"
+version = "1.2.38"
source = { editable = "." }
dependencies = [
{ name = "aiohttp" },