Use Claude Code, Codex, VSCode, Pi, and OpenCode (and 6 other harnesses) for free (1.3B+ free tokens) from your terminal, app, IDE, or phone, and now from the browser with native browser sessions (multi-harness + multi-model) like OpenClaw (voice supported + ToS friendly)
Python
Improve smoke provider coverage and skips
Alishahryar1b867e08084dc
diff --git a/pyproject.toml b/pyproject.tomlindex 06ce0d51..1c32b357 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -118,6 +118,7 @@ markers = ["voice: voice transcription checks","contract: deterministic feature contract checks","smoke_target(name): route a smoke test behind FCC_SMOKE_TARGETS",+ "xdist_group(name): group smoke provider items under pytest-xdist --dist=loadgroup",][tool.ty.environment]diff --git a/smoke/README.md b/smoke/README.mdindex 414a43d9..871f4159 100644--- a/smoke/README.md+++ b/smoke/README.md@@ -31,6 +31,15 @@ $env:FCC_LIVE_SMOKE = "1"uv run pytest smoke -n 0 -s --tb=short```+Provider smoke scenarios can run providers in parallel while preserving+sequential execution within each provider:++```powershell+$env:FCC_LIVE_SMOKE = "1"+$env:FCC_SMOKE_TARGETS = "providers"+uv run pytest smoke -n auto --dist=loadgroup -s --tb=short+```+Provider product E2E runs once per configured provider, independent of `MODEL`,`MODEL_OPUS`, `MODEL_SONNET`, and `MODEL_HAIKU`. Defaults come from the providercatalog/docs and can be overridden with `FCC_SMOKE_MODEL_<PROVIDER>`, for example@@ -100,7 +109,7 @@ uv run pytest smoke/product -n 0 -s --tb=short```powershell$env:FCC_LIVE_SMOKE = "1"$env:FCC_SMOKE_TARGETS = "openrouter_free_cli"-$env:FCC_SMOKE_OPENROUTER_FREE_MODELS = "nvidia/nemotron-3-super-120b-a12b:free,openai/gpt-oss-120b:free,minimax/minimax-m2.5:free,inclusionai/ring-2.6-1t:free,poolside/laguna-m.1:free"+$env:FCC_SMOKE_OPENROUTER_FREE_MODELS = "nvidia/nemotron-3-super-120b-a12b:free,openai/gpt-oss-120b:free,poolside/laguna-m.1:free"uv run pytest smoke/product -n 0 -s --tb=short```@@ -154,10 +163,9 @@ locked.Smoke artifacts are written to `.smoke-results/` and redact env values whosenames contain `KEY`, `TOKEN`, `SECRET`, `WEBHOOK`, or `AUTH`.-- `missing_env`: required credentials, binary, provider config, local server, or- opt-in flag is absent.-- `upstream_unavailable`: a real provider, bot API, or local model server is not- reachable.+- `missing_env`: required credentials, binary, provider config, local provider+ server/model, or opt-in flag is absent.+- `upstream_unavailable`: a real provider or bot API is not reachable.- `probe_timeout`: the smoke driver reached the target, but the CLI/probe didnot complete within the smoke timeout.- `product_failure`: the app accepted the scenario but returned the wrong shape,diff --git a/smoke/conftest.py b/smoke/conftest.pyindex a10e413e..264324aa 100644--- a/smoke/conftest.py+++ b/smoke/conftest.py@@ -1,13 +1,26 @@from __future__ import annotationsfrom collections.abc import Iterator+from typing import Anyimport pytest-from smoke.lib.config import SmokeConfig, auth_headers+from smoke.lib.config import ProviderModel, SmokeConfig, auth_headersfrom smoke.lib.report import SmokeReportfrom smoke.lib.server import RunningServer, start_server+DISABLED_PROVIDER_MODEL = ProviderModel(+ provider="smoke_disabled",+ full_model="smoke_disabled/smoke-disabled",+ source="smoke_disabled",+)+++def pytest_generate_tests(metafunc: pytest.Metafunc) -> None:+ if "provider_model" in metafunc.fixturenames:+ config = SmokeConfig.load()+ metafunc.parametrize("provider_model", provider_model_params(config))+def pytest_collection_modifyitems(items: list[pytest.Item]) -> None:if SmokeConfig.load().live:@@ -74,4 +87,42 @@ def smoke_headers() -> dict[str, str]:return auth_headers()+def provider_model_params(config: SmokeConfig) -> list[Any]:+ """Return provider params grouped for pytest-xdist ``--dist=loadgroup``."""+ if not config.live:+ return [_disabled_provider_param("set FCC_LIVE_SMOKE=1 to run provider smoke")]++ models = config.provider_smoke_models()+ if not models:+ return [_disabled_provider_param("missing_env: no configured provider smoke")]++ return [+ pytest.param(+ model,+ id=provider_model_id(model),+ marks=pytest.mark.xdist_group(provider_xdist_group(model)),+ )+ for model in models+ ]+++def _disabled_provider_param(reason: str) -> Any:+ return pytest.param(+ DISABLED_PROVIDER_MODEL,+ id=provider_model_id(DISABLED_PROVIDER_MODEL),+ marks=(+ pytest.mark.skip(reason=reason),+ pytest.mark.xdist_group(provider_xdist_group(DISABLED_PROVIDER_MODEL)),+ ),+ )+++def provider_model_id(provider_model: ProviderModel) -> str:+ return provider_model.provider+++def provider_xdist_group(provider_model: ProviderModel) -> str:+ return f"provider:{provider_model.provider}"++_REPORT: SmokeReport | None = Nonediff --git a/smoke/lib/config.py b/smoke/lib/config.pyindex f13abc5a..0b58fddb 100644--- a/smoke/lib/config.py+++ b/smoke/lib/config.py@@ -43,7 +43,7 @@ SECRET_KEY_PARTS = ("KEY", "TOKEN", "SECRET", "WEBHOOK", "AUTH")PROVIDER_SMOKE_DEFAULT_MODELS: dict[str, str] = {"nvidia_nim": "nvidia_nim/nvidia/nemotron-3-super-120b-a12b",- "open_router": "open_router/openrouter/free",+ "open_router": "open_router/moonshotai/kimi-k2.6:free","mistral": "mistral/devstral-small-latest","mistral_codestral": "mistral_codestral/codestral-latest","deepseek": "deepseek/deepseek-v4-pro",@@ -71,8 +71,6 @@ NVIDIA_NIM_CLI_DEFAULT_MODELS: tuple[str, ...] = (OPENROUTER_FREE_CLI_DEFAULT_MODELS: tuple[str, ...] = ("nvidia/nemotron-3-super-120b-a12b:free","openai/gpt-oss-120b:free",- "minimax/minimax-m2.5:free",- "inclusionai/ring-2.6-1t:free","poolside/laguna-m.1:free",)diff --git a/smoke/lib/local_providers.py b/smoke/lib/local_providers.pynew file mode 100644index 00000000..ba2be5d3--- /dev/null+++ b/smoke/lib/local_providers.py@@ -0,0 +1,85 @@+"""Helpers for local-provider smoke availability checks."""++from __future__ import annotations++from urllib.parse import urljoin++import httpx+import pytest++LOCAL_PROVIDER_PROBE_TIMEOUT_S = 1.5+++def first_local_provider_model_id(+ provider: str,+ base_url: str,+ *,+ timeout_s: float,+) -> str:+ """Return the first local model id, or skip when the local server is absent."""+ base_url = base_url.strip()+ if not base_url:+ pytest.skip(f"missing_env: {provider} base URL is not configured")++ timeout = min(timeout_s, LOCAL_PROVIDER_PROBE_TIMEOUT_S)+ if provider == "ollama":+ return _first_ollama_model_id(base_url, timeout_s=timeout)+ return _first_openai_compatible_model_id(+ provider,+ base_url,+ timeout_s=timeout,+ )+++def _first_openai_compatible_model_id(+ provider: str,+ base_url: str,+ *,+ timeout_s: float,+) -> str:+ models_url = urljoin(base_url.rstrip("/") + "/", "models")+ response = _get_local_provider_response(provider, models_url, timeout_s=timeout_s)+ assert response.status_code == 200, response.text+ payload = response.json()+ data = payload.get("data") if isinstance(payload, dict) else None+ if isinstance(data, list):+ for item in data:+ if isinstance(item, dict) and isinstance(item.get("id"), str):+ return item["id"]+ pytest.skip(f"missing_env: {provider} local server has no loaded models")+ pytest.fail("product_failure: local /models did not expose a model id")+++def _first_ollama_model_id(base_url: str, *, timeout_s: float) -> str:+ tags_url = f"{base_url.rstrip('/')}/api/tags"+ response = _get_local_provider_response("ollama", tags_url, timeout_s=timeout_s)+ assert response.status_code == 200, response.text+ payload = response.json()+ models = payload.get("models") if isinstance(payload, dict) else None+ if isinstance(models, list):+ for item in models:+ if isinstance(item, dict) and isinstance(item.get("name"), str):+ return item["name"]+ pytest.skip("missing_env: ollama local server has no pulled models")+ pytest.fail("product_failure: ollama /api/tags did not expose models")+++def _get_local_provider_response(+ provider: str,+ url: str,+ *,+ timeout_s: float,+) -> httpx.Response:+ try:+ response = httpx.get(url, timeout=timeout_s)+ except httpx.TimeoutException as exc:+ pytest.skip(f"missing_env: {provider} local server is not reachable: {exc}")+ except httpx.NetworkError as exc:+ pytest.skip(f"missing_env: {provider} local server is not running: {exc}")++ if response.status_code in {404, 405, 502, 503}:+ pytest.skip(+ f"missing_env: {provider} local server is not available at {url}: "+ f"HTTP {response.status_code}"+ )+ return responsediff --git a/smoke/lib/report.py b/smoke/lib/report.pyindex ea3b0b38..8f499c46 100644--- a/smoke/lib/report.py+++ b/smoke/lib/report.py@@ -71,6 +71,8 @@ def classify_outcome(*, nodeid: str, outcome: str, detail: str) -> str:if outcome == "skipped":if "smoke target disabled" in text:return "target_disabled"+ if "missing_env" in text:+ return "missing_env"if any(marker in textfor marker in (diff --git a/smoke/lib/skips.py b/smoke/lib/skips.pyindex 1cb1e29a..73d66426 100644--- a/smoke/lib/skips.py+++ b/smoke/lib/skips.py@@ -5,7 +5,7 @@ from __future__ import annotationsimport httpximport pytest-from core.anthropic.stream_contracts import SSEEvent+from core.anthropic.stream_contracts import SSEEvent, text_contentUPSTREAM_UNAVAILABLE_MARKERS = ("connection refused",@@ -50,6 +50,10 @@ def skip_if_upstream_unavailable_exception(exc: Exception) -> None:def skip_if_upstream_unavailable_events(events: list[SSEEvent]) -> None:+ text = text_content(events)+ if is_upstream_unavailable_text(text):+ skip_upstream_unavailable(text[:500])+for event in events:if getattr(event, "event", None) != "error":continuediff --git a/smoke/prereq/test_local_provider_endpoints_prereq_live.py b/smoke/prereq/test_local_provider_endpoints_prereq_live.pyindex 43f59c22..affedf9b 100644--- a/smoke/prereq/test_local_provider_endpoints_prereq_live.py+++ b/smoke/prereq/test_local_provider_endpoints_prereq_live.py@@ -1,86 +1,36 @@from __future__ import annotations-import httpximport pytestfrom smoke.lib.config import SmokeConfig-from smoke.lib.skips import skip_if_upstream_unavailable_exception+from smoke.lib.local_providers import first_local_provider_model_id@pytest.mark.live@pytest.mark.smoke_target("lmstudio")def test_lmstudio_models_endpoint_when_available(smoke_config: SmokeConfig) -> None:- _assert_models_endpoint(+ first_local_provider_model_id(+ "lmstudio",smoke_config.settings.lm_studio_base_url,timeout_s=smoke_config.timeout_s,- provider_name="LM Studio",)@pytest.mark.live@pytest.mark.smoke_target("llamacpp")def test_llamacpp_models_endpoint_when_available(smoke_config: SmokeConfig) -> None:- _assert_models_endpoint(+ first_local_provider_model_id(+ "llamacpp",smoke_config.settings.llamacpp_base_url,timeout_s=smoke_config.timeout_s,- provider_name="llama.cpp",)@pytest.mark.live@pytest.mark.smoke_target("ollama")def test_ollama_models_endpoint_when_available(smoke_config: SmokeConfig) -> None:- _assert_ollama_tags_endpoint(- smoke_config.settings.ollama_base_url, timeout_s=smoke_config.timeout_s+ first_local_provider_model_id(+ "ollama",+ smoke_config.settings.ollama_base_url,+ timeout_s=smoke_config.timeout_s,)---def _assert_models_endpoint(- base_url: str, *, timeout_s: float, provider_name: str-) -> None:- url = f"{base_url.rstrip('/')}/models"- try:- response = httpx.get(url, timeout=timeout_s)- except Exception as exc:- skip_if_upstream_unavailable_exception(exc)- raise-- if response.status_code in {404, 405, 502, 503}:- pytest.skip(- f"upstream_unavailable: {provider_name} models endpoint "- f"{url} returned HTTP {response.status_code}"- )-- assert response.status_code == 200, response.text- payload = response.json()- data = payload.get("data")- if isinstance(data, list) and data:- return- if isinstance(data, list):- pytest.skip(f"upstream_unavailable: {provider_name} has no local models")- assert isinstance(data, list), payload---def _assert_ollama_tags_endpoint(base_url: str, *, timeout_s: float) -> None:- url = f"{_ollama_root_url(base_url)}/api/tags"- try:- response = httpx.get(url, timeout=timeout_s)- except Exception as exc:- skip_if_upstream_unavailable_exception(exc)- raise-- if response.status_code in {404, 405, 502, 503}:- pytest.skip(- f"upstream_unavailable: Ollama tags endpoint {url} "- f"returned HTTP {response.status_code}"- )-- assert response.status_code == 200, response.text- models = response.json().get("models")- if isinstance(models, list) and models:- return- pytest.skip("upstream_unavailable: Ollama has no pulled models")---def _ollama_root_url(base_url: str) -> str:- return base_url.rstrip("/")diff --git a/smoke/prereq/test_provider_prereq_live.py b/smoke/prereq/test_provider_prereq_live.pyindex c5269f10..b08e808d 100644--- a/smoke/prereq/test_provider_prereq_live.py+++ b/smoke/prereq/test_provider_prereq_live.py@@ -10,7 +10,7 @@ from core.anthropic.stream_contracts import (text_content,thinking_content,)-from smoke.lib.config import SmokeConfig, auth_headers+from smoke.lib.config import ProviderModel, SmokeConfig, auth_headersfrom smoke.lib.e2e import ProviderMatrixDriverfrom smoke.lib.http import collect_message_stream, message_payloadfrom smoke.lib.server import start_server@@ -45,41 +45,35 @@ def test_mixed_provider_model_mapping_when_configured(def test_configured_provider_models_stream_successfully(- smoke_config: SmokeConfig,+ smoke_config: SmokeConfig, provider_model: ProviderModel) -> None:- models = ProviderMatrixDriver(smoke_config).provider_smoke_models()-- failures: list[str] = []- for provider_model in models:- try:- with start_server(+ try:+ with start_server(+ smoke_config,+ env_overrides={+ "MODEL": provider_model.full_model,+ "MESSAGING_PLATFORM": "none",+ },+ name=f"provider-{provider_model.provider}",+ ) as server:+ events = collect_message_stream(+ server,+ message_payload(smoke_config.prompt, model="fcc-smoke-default"),smoke_config,- env_overrides={- "MODEL": provider_model.full_model,- "MESSAGING_PLATFORM": "none",- },- name=f"provider-{provider_model.provider}",- ) as server:- events = collect_message_stream(- server,- message_payload(smoke_config.prompt, model="fcc-smoke-default"),- smoke_config,- )- skip_if_upstream_unavailable_events(events)- assert_anthropic_stream_contract(events)- has_text = bool(text_content(events).strip())- has_thinking = bool(thinking_content(events).strip())- assert has_text or has_thinking, (- "provider returned no visible text or thinking content"- )- except Exception as exc:- skip_if_upstream_unavailable_exception(exc)- failures.append(- f"{provider_model.source}={provider_model.full_model}: "- f"{type(exc).__name__}: {exc}")-- assert not failures, "\n".join(failures)+ skip_if_upstream_unavailable_events(events)+ assert_anthropic_stream_contract(events)+ has_text = bool(text_content(events).strip())+ has_thinking = bool(thinking_content(events).strip())+ assert has_text or has_thinking, (+ "provider returned no visible text or thinking content"+ )+ except Exception as exc:+ skip_if_upstream_unavailable_exception(exc)+ raise AssertionError(+ f"{provider_model.source}={provider_model.full_model}: "+ f"{type(exc).__name__}: {exc}"+ ) from exc@pytest.mark.smoke_target("rate_limit")diff --git a/smoke/product/test_local_provider_product_live.py b/smoke/product/test_local_provider_product_live.pyindex a8fa519e..5d3797ce 100644--- a/smoke/product/test_local_provider_product_live.py+++ b/smoke/product/test_local_provider_product_live.py@@ -1,12 +1,10 @@from __future__ import annotations-from urllib.parse import urljoin--import httpximport pytestfrom smoke.lib.config import SmokeConfigfrom smoke.lib.e2e import ConversationDriver, SmokeServerDriver, assert_product_stream+from smoke.lib.local_providers import first_local_provider_model_idpytestmark = [pytest.mark.live]@@ -44,13 +42,10 @@ def _local_native_messages_e2e(provider: str,base_url: str,) -> None:- if not base_url.strip():- pytest.skip(f"missing_env: {provider} base URL is not configured")-- model_id = (- _first_ollama_tag_model_id(base_url)- if provider == "ollama"- else (_first_non_ollama_model_id(provider, base_url))+ model_id = first_local_provider_model_id(+ provider,+ base_url,+ timeout_s=smoke_config.timeout_s,)with SmokeServerDriver(@@ -63,46 +58,3 @@ def _local_native_messages_e2e()assert_product_stream(turn.events)---def _first_non_ollama_model_id(provider: str, base_url: str) -> str:- models_url = urljoin(base_url.rstrip("/") + "/", "models")- try:- response = httpx.get(models_url, timeout=5)- except httpx.ConnectError as exc:- pytest.skip(f"upstream_unavailable: {provider} models endpoint: {exc}")- except httpx.TimeoutException as exc:- pytest.skip(f"upstream_unavailable: {provider} models endpoint: {exc}")- assert response.status_code == 200, response.text- payload = response.json()- data = payload.get("data") if isinstance(payload, dict) else None- if isinstance(data, list):- for item in data:- if isinstance(item, dict) and isinstance(item.get("id"), str):- return item["id"]- pytest.skip(f"upstream_unavailable: {provider} has no local models")- pytest.fail("product_failure: local /models did not expose a model id")---def _first_ollama_tag_model_id(base_url: str) -> str:- tags_url = f"{_ollama_root_url(base_url)}/api/tags"- try:- response = httpx.get(tags_url, timeout=5)- except httpx.ConnectError as exc:- pytest.skip(f"upstream_unavailable: ollama tags endpoint: {exc}")- except httpx.TimeoutException as exc:- pytest.skip(f"upstream_unavailable: ollama tags endpoint: {exc}")-- assert response.status_code == 200, response.text- payload = response.json()- models = payload.get("models") if isinstance(payload, dict) else None- if isinstance(models, list):- for item in models:- if isinstance(item, dict) and isinstance(item.get("name"), str):- return item["name"]- pytest.skip("upstream_unavailable: ollama has no pulled models")- pytest.fail("product_failure: ollama /api/tags did not expose models")---def _ollama_root_url(base_url: str) -> str:- return base_url.rstrip("/")diff --git a/smoke/product/test_provider_product_live.py b/smoke/product/test_provider_product_live.pyindex 593d6412..6bcee620 100644--- a/smoke/product/test_provider_product_live.py+++ b/smoke/product/test_provider_product_live.py@@ -7,6 +7,7 @@ import pytestfrom config.provider_catalog import PROVIDER_CATALOGfrom core.anthropic.stream_contracts import (+ SSEEvent,assert_anthropic_stream_contract,parse_sse_lines,text_content,@@ -42,51 +43,74 @@ def test_model_mapping_matrix_e2e(smoke_config: SmokeConfig) -> None:assert model.model_name-def test_provider_text_multiturn_e2e(smoke_config: SmokeConfig) -> None:- _run_for_each_provider(smoke_config, _scenario_text_multiturn)+def test_provider_text_multiturn_e2e(+ smoke_config: SmokeConfig, provider_model: ProviderModel+) -> None:+ _run_provider_scenario(smoke_config, provider_model, _scenario_text_multiturn)-def test_provider_adaptive_thinking_history_e2e(smoke_config: SmokeConfig) -> None:- _run_for_each_provider(smoke_config, _scenario_adaptive_thinking_history)+def test_provider_adaptive_thinking_history_e2e(+ smoke_config: SmokeConfig, provider_model: ProviderModel+) -> None:+ _run_provider_scenario(+ smoke_config, provider_model, _scenario_adaptive_thinking_history+ )-def test_provider_interleaved_thinking_tool_e2e(smoke_config: SmokeConfig) -> None:- _run_for_each_provider(smoke_config, _scenario_interleaved_history)+def test_provider_interleaved_thinking_tool_e2e(+ smoke_config: SmokeConfig, provider_model: ProviderModel+) -> None:+ _run_provider_scenario(smoke_config, provider_model, _scenario_interleaved_history)@pytest.mark.smoke_target("tools")-def test_provider_tool_use_then_text_history_e2e(smoke_config: SmokeConfig) -> None:+def test_provider_tool_use_then_text_history_e2e(+ smoke_config: SmokeConfig, provider_model: ProviderModel+) -> None:"""OpenAI-compatible path: history with tool_use + assistant text after tool (issue #206)."""- _run_for_each_provider(smoke_config, _scenario_tool_use_then_text_in_history)+ _run_provider_scenario(+ smoke_config, provider_model, _scenario_tool_use_then_text_in_history+ )@pytest.mark.smoke_target("tools")-def test_provider_tool_result_continuation_e2e(smoke_config: SmokeConfig) -> None:- _run_for_each_provider(smoke_config, _scenario_tool_result_continuation)+def test_provider_tool_result_continuation_e2e(+ smoke_config: SmokeConfig, provider_model: ProviderModel+) -> None:+ _run_provider_scenario(+ smoke_config, provider_model, _scenario_tool_result_continuation+ )@pytest.mark.smoke_target("tools")def test_gemini_thought_signature_tool_continuation_e2e(- smoke_config: SmokeConfig,+ smoke_config: SmokeConfig, provider_model: ProviderModel) -> None:- models = [- model- for model in ProviderMatrixDriver(smoke_config).provider_smoke_models()- if model.provider == "gemini"- ]- if not models:- pytest.skip("missing_env: gemini is not configured")- _scenario_gemini_thought_signature_tool_continuation(smoke_config, models[0])+ if provider_model.provider != "gemini":+ pytest.skip("gemini-specific smoke scenario")+ _run_provider_scenario(+ smoke_config,+ provider_model,+ _scenario_gemini_thought_signature_tool_continuation,+ )@pytest.mark.smoke_target("tools")-def test_provider_reasoning_tool_continuation_e2e(smoke_config: SmokeConfig) -> None:- _run_for_each_thinking_provider(smoke_config, _scenario_reasoning_tool_continuation)+def test_provider_reasoning_tool_continuation_e2e(+ smoke_config: SmokeConfig, provider_model: ProviderModel+) -> None:+ if not _provider_smoke_thinking_enabled(smoke_config, provider_model):+ pytest.skip(f"{provider_model.provider} smoke model does not enable thinking")+ _run_provider_scenario(+ smoke_config, provider_model, _scenario_reasoning_tool_continuation+ )@pytest.mark.smoke_target("rate_limit")-def test_provider_disconnect_e2e(smoke_config: SmokeConfig) -> None:- _run_for_each_provider(smoke_config, _scenario_disconnect)+def test_provider_disconnect_e2e(+ smoke_config: SmokeConfig, provider_model: ProviderModel+) -> None:+ _run_provider_scenario(smoke_config, provider_model, _scenario_disconnect)def test_provider_error_e2e(smoke_config: SmokeConfig) -> None:@@ -147,42 +171,36 @@ def test_openrouter_native_e2e(smoke_config: SmokeConfig) -> None:"thinking": {"type": "adaptive", "budget_tokens": 1024},})- assert_product_stream(turn.events)+ _assert_provider_product_stream(turn.events)-def _run_for_each_provider(smoke_config: SmokeConfig, scenario) -> None:- failures: list[str] = []- for provider_model in ProviderMatrixDriver(smoke_config).provider_smoke_models():- try:- scenario(smoke_config, provider_model)- except Exception as exc:- skip_if_upstream_unavailable_exception(exc)- failures.append(- f"{provider_model.source}={provider_model.full_model}: "- f"{type(exc).__name__}: {exc}"- )- assert not failures, "\n".join(failures)+def _run_provider_scenario(+ smoke_config: SmokeConfig,+ provider_model: ProviderModel,+ scenario,+) -> None:+ try:+ scenario(smoke_config, provider_model)+ except Exception as exc:+ skip_if_upstream_unavailable_exception(exc)+ raise AssertionError(+ f"{provider_model.source}={provider_model.full_model}: "+ f"{type(exc).__name__}: {exc}"+ ) from exc-def _run_for_each_thinking_provider(smoke_config: SmokeConfig, scenario) -> None:- failures: list[str] = []- models = [- provider_model- for provider_model in ProviderMatrixDriver(smoke_config).provider_smoke_models()- if _provider_smoke_thinking_enabled(smoke_config, provider_model)- ]- if not models:- pytest.skip("missing_env: no thinking-capable provider smoke model configured")- for provider_model in models:- try:- scenario(smoke_config, provider_model)- except Exception as exc:- skip_if_upstream_unavailable_exception(exc)- failures.append(- f"{provider_model.source}={provider_model.full_model}: "- f"{type(exc).__name__}: {exc}"- )- assert not failures, "\n".join(failures)+def _assert_provider_product_stream(events: list[SSEEvent]) -> None:+ skip_if_upstream_unavailable_events(events)+ assert_product_stream(events)+++def _tool_use_blocks_or_skip(+ events: list[SSEEvent], message: str+) -> list[dict[str, Any]]:+ skip_if_upstream_unavailable_events(events)+ blocks = tool_use_blocks(events)+ assert blocks, message+ return blocksdef _provider_smoke_thinking_enabled(@@ -202,8 +220,8 @@ def _scenario_text_multiturn(driver = ConversationDriver(server, smoke_config)first = driver.ask("Reply with one short sentence.")second = driver.ask("Reply with a different short sentence.")- assert_product_stream(first.events)- assert_product_stream(second.events)+ _assert_provider_product_stream(first.events)+ _assert_provider_product_stream(second.events)def _scenario_adaptive_thinking_history(@@ -228,7 +246,7 @@ def _scenario_adaptive_thinking_history(}with _server_for_provider(smoke_config, provider_model, "adaptive") as server:turn = ConversationDriver(server, smoke_config).stream(payload)- assert_product_stream(turn.events)+ _assert_provider_product_stream(turn.events)def _scenario_interleaved_history(@@ -268,7 +286,7 @@ def _scenario_interleaved_history(}with _server_for_provider(smoke_config, provider_model, "interleaved") as server:turn = ConversationDriver(server, smoke_config).stream(payload)- assert_product_stream(turn.events)+ _assert_provider_product_stream(turn.events)def _scenario_tool_use_then_text_in_history(@@ -314,7 +332,7 @@ def _scenario_tool_use_then_text_in_history(}with _server_for_provider(smoke_config, provider_model, "tool-206") as server:turn = ConversationDriver(server, smoke_config).stream(payload)- assert_product_stream(turn.events)+ _assert_provider_product_stream(turn.events)def _scenario_tool_result_continuation(@@ -333,8 +351,9 @@ def _scenario_tool_result_continuation(with _server_for_provider(smoke_config, provider_model, "tool") as server:driver = ConversationDriver(server, smoke_config)first = driver.stream(first_payload)- tool_uses = tool_use_blocks(first.events)- assert tool_uses, "provider did not emit a tool_use block"+ tool_uses = _tool_use_blocks_or_skip(+ first.events, "provider did not emit a tool_use block"+ )tool_use = tool_uses[0]second_payload = {"model": "claude-sonnet-4-5-20250929",@@ -356,8 +375,8 @@ def _scenario_tool_result_continuation("tools": [echo_tool_schema()],}second = driver.stream(second_payload)- assert_product_stream(first.events)- assert_product_stream(second.events)+ _assert_provider_product_stream(first.events)+ _assert_provider_product_stream(second.events)def _scenario_gemini_thought_signature_tool_continuation(@@ -378,8 +397,9 @@ def _scenario_gemini_thought_signature_tool_continuation() as server:driver = ConversationDriver(server, smoke_config)first = driver.stream(first_payload)- tool_uses = tool_use_blocks(first.events)- assert tool_uses, "gemini did not emit a tool_use block"+ tool_uses = _tool_use_blocks_or_skip(+ first.events, "gemini did not emit a tool_use block"+ )tool_use = tool_uses[0]signature = _gemini_tool_thought_signature(tool_use)assert signature, (@@ -406,8 +426,8 @@ def _scenario_gemini_thought_signature_tool_continuation("thinking": {"type": "adaptive", "budget_tokens": 1024},}second = driver.stream(second_payload)- assert_product_stream(first.events)- assert_product_stream(second.events)+ _assert_provider_product_stream(first.events)+ _assert_provider_product_stream(second.events)def _gemini_tool_thought_signature(tool_use: dict[str, Any]) -> str | None:@@ -457,7 +477,7 @@ def _scenario_reasoning_tool_continuation(}with _server_for_provider(smoke_config, provider_model, "reasoning-tool") as server:turn = ConversationDriver(server, smoke_config).stream(payload)- assert_product_stream(turn.events)+ _assert_provider_product_stream(turn.events)def _scenario_disconnect(@@ -483,8 +503,7 @@ def _scenario_disconnect(followup = ConversationDriver(server, smoke_config).ask("Reply with one short sentence.")- skip_if_upstream_unavailable_events(followup.events)- assert_product_stream(followup.events)+ _assert_provider_product_stream(followup.events)def _server_for_provider(diff --git a/tests/contracts/test_local_provider_smoke.py b/tests/contracts/test_local_provider_smoke.pynew file mode 100644index 00000000..a5ea7b32--- /dev/null+++ b/tests/contracts/test_local_provider_smoke.py@@ -0,0 +1,89 @@+from __future__ import annotations++from dataclasses import dataclass++import httpx+import pytest++from smoke.lib.local_providers import first_local_provider_model_id+++@dataclass(slots=True)+class FakeResponse:+ status_code: int+ payload: dict+ text: str = ""++ def json(self) -> dict:+ return self.payload+++def test_local_provider_openai_models_returns_first_id(monkeypatch) -> None:+ calls: list[tuple[str, float]] = []++ def fake_get(url: str, *, timeout: float) -> FakeResponse:+ calls.append((url, timeout))+ return FakeResponse(200, {"data": [{"id": "local-model"}]})++ monkeypatch.setattr("smoke.lib.local_providers.httpx.get", fake_get)++ model = first_local_provider_model_id(+ "lmstudio",+ "http://127.0.0.1:1234/v1",+ timeout_s=45,+ )++ assert model == "local-model"+ assert calls == [("http://127.0.0.1:1234/v1/models", 1.5)]+++def test_local_provider_ollama_tags_returns_first_name(monkeypatch) -> None:+ def fake_get(url: str, *, timeout: float) -> FakeResponse:+ assert url == "http://127.0.0.1:11434/api/tags"+ assert timeout == 1.5+ return FakeResponse(200, {"models": [{"name": "llama3.1"}]})++ monkeypatch.setattr("smoke.lib.local_providers.httpx.get", fake_get)++ assert (+ first_local_provider_model_id(+ "ollama",+ "http://127.0.0.1:11434",+ timeout_s=45,+ )+ == "llama3.1"+ )+++def test_local_provider_not_running_is_missing_env_skip(monkeypatch) -> None:+ def fake_get(url: str, *, timeout: float) -> FakeResponse:+ raise httpx.ConnectError("refused")++ monkeypatch.setattr("smoke.lib.local_providers.httpx.get", fake_get)++ with pytest.raises(pytest.skip.Exception) as excinfo:+ first_local_provider_model_id(+ "llamacpp",+ "http://127.0.0.1:8080/v1",+ timeout_s=45,+ )++ assert "missing_env: llamacpp local server is not running" in str(excinfo.value)+++def test_local_provider_empty_model_list_is_missing_env_skip(monkeypatch) -> None:+ def fake_get(url: str, *, timeout: float) -> FakeResponse:+ return FakeResponse(200, {"data": []})++ monkeypatch.setattr("smoke.lib.local_providers.httpx.get", fake_get)++ with pytest.raises(pytest.skip.Exception) as excinfo:+ first_local_provider_model_id(+ "lmstudio",+ "http://127.0.0.1:1234/v1",+ timeout_s=45,+ )++ assert "missing_env: lmstudio local server has no loaded models" in str(+ excinfo.value+ )diff --git a/tests/contracts/test_nvidia_nim_cli_matrix.py b/tests/contracts/test_nvidia_nim_cli_matrix.pyindex 4666de3e..ae45b4d6 100644--- a/tests/contracts/test_nvidia_nim_cli_matrix.py+++ b/tests/contracts/test_nvidia_nim_cli_matrix.py@@ -342,8 +342,8 @@ def test_cli_matrix_structured_provider_error_is_upstream_unavailable(duration_s=0.1,)outcome = make_outcome(- model="minimax/minimax-m2.5:free",- full_model="open_router/minimax/minimax-m2.5:free",+ model="poolside/laguna-m.1:free",+ full_model="open_router/poolside/laguna-m.1:free",source="openrouter_free_cli_default",feature="tool_use_roundtrip",marker="FCC_OPENROUTER_FREE_TOOL",diff --git a/tests/contracts/test_smoke_config.py b/tests/contracts/test_smoke_config.pyindex 257eb2c2..c57aa8b0 100644--- a/tests/contracts/test_smoke_config.py+++ b/tests/contracts/test_smoke_config.py@@ -3,6 +3,11 @@ from __future__ import annotationsfrom pathlib import Pathfrom types import SimpleNamespace+from smoke.conftest import (+ DISABLED_PROVIDER_MODEL,+ provider_model_params,+ provider_xdist_group,+)from smoke.lib.config import (ALL_TARGETS,DEFAULT_TARGETS,@@ -110,6 +115,19 @@ def test_provider_smoke_models_cover_configured_providers_independent_of_model_massert models[0].source == "provider_default"+def test_openrouter_provider_smoke_uses_concrete_free_model(monkeypatch) -> None:+ monkeypatch.delenv("FCC_SMOKE_MODEL_OPEN_ROUTER", raising=False)+ config = _smoke_config(+ settings=_settings(open_router_api_key="openrouter-key", ollama_base_url="")+ )++ models = config.provider_smoke_models()++ assert [model.provider for model in models] == ["open_router"]+ assert models[0].full_model == "open_router/moonshotai/kimi-k2.6:free"+ assert models[0].source == "provider_default"++def test_wafer_provider_configuration_uses_api_key(monkeypatch) -> None:monkeypatch.delenv("FCC_SMOKE_MODEL_WAFER", raising=False)config = _smoke_config(@@ -202,6 +220,41 @@ def test_provider_smoke_matrix_filters_provider_catalog(monkeypatch) -> None:]+def test_provider_smoke_collection_params_are_grouped_by_provider(+ monkeypatch,+) -> None:+ monkeypatch.delenv("FCC_SMOKE_MODEL_DEEPSEEK", raising=False)+ monkeypatch.delenv("FCC_SMOKE_MODEL_NVIDIA_NIM", raising=False)+ config = _smoke_config(+ live=True,+ settings=_settings(+ deepseek_api_key="deepseek-key",+ nvidia_nim_api_key="nim-key",+ ollama_base_url="",+ ),+ )++ params = provider_model_params(config)++ assert [param.id for param in params] == ["nvidia_nim", "deepseek"]+ groups = [+ mark.args[0]+ for param in params+ for mark in param.marks+ if mark.name == "xdist_group"+ ]+ assert groups == ["provider:nvidia_nim", "provider:deepseek"]+++def test_provider_smoke_collection_uses_disabled_placeholder_when_not_live() -> None:+ config = _smoke_config(live=False, settings=_settings(ollama_base_url=""))++ params = provider_model_params(config)++ assert [param.values[0] for param in params] == [DISABLED_PROVIDER_MODEL]+ assert provider_xdist_group(DISABLED_PROVIDER_MODEL) == "provider:smoke_disabled"++def test_provider_smoke_includes_local_provider_when_model_mapping_uses_it(monkeypatch,) -> None:diff --git a/tests/contracts/test_smoke_tiers.py b/tests/contracts/test_smoke_tiers.pyindex ee92416b..85a4855b 100644--- a/tests/contracts/test_smoke_tiers.py+++ b/tests/contracts/test_smoke_tiers.py@@ -3,8 +3,12 @@ from __future__ import annotationsimport jsonfrom pathlib import Path+import pytest++from core.anthropic.stream_contracts import SSEEventfrom smoke.lib.report import classify_outcomefrom smoke.lib.report_summary import format_summary, summarize_reports+from smoke.lib.skips import skip_if_upstream_unavailable_eventsdef test_smoke_readme_uses_env_gated_serial_commands() -> None:@@ -43,3 +47,44 @@ def test_target_disabled_skip_is_not_missing_env() -> None:)assert classification == "target_disabled"+++def test_explicit_missing_env_skip_wins_over_network_words() -> None:+ classification = classify_outcome(+ nodeid="smoke/prereq/test_local_provider_endpoints_prereq_live.py::"+ "test_ollama_endpoint_prereq_live",+ outcome="skipped",+ detail="Skipped: missing_env: ollama local server is not reachable: timed out",+ )++ assert classification == "missing_env"+++def test_provider_error_text_stream_is_upstream_unavailable_skip() -> None:+ events = [+ SSEEvent("message_start", {"message": {"content": []}}, ""),+ SSEEvent(+ "content_block_start",+ {"index": 0, "content_block": {"type": "text", "text": ""}},+ "",+ ),+ SSEEvent(+ "content_block_delta",+ {+ "index": 0,+ "delta": {+ "type": "text_delta",+ "text": "Upstream provider OPENROUTER returned HTTP 429.",+ },+ },+ "",+ ),+ SSEEvent("content_block_stop", {"index": 0}, ""),+ SSEEvent("message_delta", {"delta": {"stop_reason": "end_turn"}}, ""),+ SSEEvent("message_stop", {}, ""),+ ]++ with pytest.raises(pytest.skip.Exception) as excinfo:+ skip_if_upstream_unavailable_events(events)++ assert "upstream_unavailable" in str(excinfo.value)diff --git a/uv.lock b/uv.lockindex 8b90a3a0..6dd20cf6 100644--- a/uv.lock+++ b/uv.lock@@ -554,7 +554,7 @@ wheels = [[[package]]name = "free-claude-code"-version = "2.0.0"+version = "1.2.38"source = { editable = "." }dependencies = [{ name = "aiohttp" },