Use Claude Code, Codex, VSCode, Pi, and OpenCode (and 6 other harnesses) for free (1.3B+ free tokens) from your terminal, app, IDE, or phone, and now from the browser with native browser sessions (multi-harness + multi-model) like OpenClaw (voice supported + ToS friendly)
Python
Add Claude CLI smoke matrices
Alishahryar1de8e90289924
diff --git a/.env.example b/.env.exampleindex 79af59da..10b98f56 100644--- a/.env.example+++ b/.env.example@@ -51,6 +51,8 @@ FCC_SMOKE_MODEL_KIMI=FCC_SMOKE_MODEL_WAFER=FCC_SMOKE_NIM_MODELS=FCC_SMOKE_NIM_EXTRA_MODELS=+FCC_SMOKE_OPENROUTER_FREE_MODELS=+FCC_SMOKE_OPENROUTER_FREE_EXTRA_MODELS=# Thinking outputdiff --git a/smoke/README.md b/smoke/README.mdindex e0208b4d..2eb36cb8 100644--- a/smoke/README.md+++ b/smoke/README.md@@ -63,6 +63,7 @@ Heavy/side-effectful targets are opt-in:| Target | Product scenarios | Required environment || --- | --- | --- || `nvidia_nim_cli` | Claude Code CLI feature matrix across NIM models | `NVIDIA_NIM_API_KEY`, Claude CLI |+| `openrouter_free_cli` | Claude Code CLI feature matrix across OpenRouter free models | `OPENROUTER_API_KEY`, Claude CLI || `telegram` | getMe, send, edit, delete, optional manual inbound | token and chat/user ID || `discord` | channel access, send, edit, delete, optional manual inbound | token and channel ID || `voice` | generated WAV through local Whisper or NVIDIA NIM transcription | `VOICE_NOTE_ENABLED=true`, `FCC_SMOKE_RUN_VOICE=1` |@@ -96,6 +97,13 @@ $env:FCC_SMOKE_NIM_MODELS = "z-ai/glm-5.1,moonshotai/kimi-k2.6,minimaxai/minimaxuv run pytest smoke/product -n 0 -s --tb=short```+```powershell+$env:FCC_LIVE_SMOKE = "1"+$env:FCC_SMOKE_TARGETS = "openrouter_free_cli"+$env:FCC_SMOKE_OPENROUTER_FREE_MODELS = "nvidia/nemotron-3-super-120b-a12b:free,openai/gpt-oss-120b:free,minimax/minimax-m2.5:free,inclusionai/ring-2.6-1t:free,poolside/laguna-m.1:free"+uv run pytest smoke/product -n 0 -s --tb=short+```+```powershell$env:FCC_LIVE_SMOKE = "1"$env:FCC_SMOKE_TARGETS = "messaging,config,extensibility"@@ -118,6 +126,10 @@ uv run pytest smoke/product -n 0 -s --tb=shortthat replace the default characterization set.- `FCC_SMOKE_NIM_EXTRA_MODELS`: optional comma-separated NVIDIA NIM CLI matrixmodels appended to the default or replacement set.+- `FCC_SMOKE_OPENROUTER_FREE_MODELS`: optional comma-separated OpenRouter free+ CLI matrix models that replace the default characterization set.+- `FCC_SMOKE_OPENROUTER_FREE_EXTRA_MODELS`: optional comma-separated OpenRouter+ free CLI matrix models appended to the default or replacement set.- `FCC_SMOKE_TIMEOUT_S`: per-request/subprocess timeout, default `45`.- `FCC_SMOKE_CLAUDE_BIN`: Claude CLI executable name, default `claude`.- `FCC_SMOKE_TELEGRAM_CHAT_ID`: Telegram chat/user ID for send/edit/delete.diff --git a/smoke/capabilities.py b/smoke/capabilities.pyindex 2a3efbe1..52ced768 100644--- a/smoke/capabilities.py+++ b/smoke/capabilities.py@@ -411,7 +411,11 @@ CAPABILITY_CONTRACTS: tuple[CapabilityContract, ...] = ("stream-json events and session id mapping","stderr/error event and process cleanup",("tests/cli/test_cli.py",),- ("test_claude_cli_prompt_when_available", "test_nvidia_nim_cli_matrix_e2e"),+ (+ "test_claude_cli_prompt_when_available",+ "test_nvidia_nim_cli_matrix_e2e",+ "test_openrouter_free_cli_matrix_e2e",+ ),),CapabilityContract("extensibility",diff --git a/smoke/features.py b/smoke/features.pyindex 4bb95faa..494fcb94 100644--- a/smoke/features.py+++ b/smoke/features.py@@ -73,11 +73,17 @@ FEATURE_INVENTORY: tuple[FeatureCoverage, ...] = ("test_api_basic_conversation_e2e","test_claude_cli_adaptive_thinking_e2e","test_nvidia_nim_cli_matrix_e2e",+ "test_openrouter_free_cli_matrix_e2e","test_vscode_protocol_e2e","test_jetbrains_protocol_e2e",),- ("api", "cli", "clients", "nvidia_nim_cli"),- ("configured provider", "FCC_SMOKE_CLAUDE_BIN for real Claude CLI"),+ ("api", "cli", "clients", "nvidia_nim_cli", "openrouter_free_cli"),+ (+ "configured provider",+ "FCC_SMOKE_CLAUDE_BIN for real Claude CLI",+ "NVIDIA_NIM_API_KEY",+ "OPENROUTER_API_KEY",+ ),"skip real CLI when binary is absent; configured providers must pass",),FeatureCoverage(@@ -386,9 +392,15 @@ FEATURE_INVENTORY: tuple[FeatureCoverage, ...] = ("test_claude_cli_adaptive_thinking_e2e","test_claude_cli_multiturn_tool_protocol_e2e","test_nvidia_nim_cli_matrix_e2e",+ "test_openrouter_free_cli_matrix_e2e",+ ),+ ("cli", "nvidia_nim_cli", "openrouter_free_cli"),+ (+ "FCC_SMOKE_CLAUDE_BIN",+ "configured provider",+ "NVIDIA_NIM_API_KEY",+ "OPENROUTER_API_KEY",),- ("cli", "nvidia_nim_cli"),- ("FCC_SMOKE_CLAUDE_BIN", "configured provider", "NVIDIA_NIM_API_KEY"),"skip only when Claude CLI binary is absent",),FeatureCoverage(diff --git a/smoke/lib/claude_cli_matrix.py b/smoke/lib/claude_cli_matrix.pynew file mode 100644index 00000000..6a27b22f--- /dev/null+++ b/smoke/lib/claude_cli_matrix.py@@ -0,0 +1,774 @@+"""Claude Code CLI characterization helpers for provider smoke matrices."""++from __future__ import annotations++import json+import os+import re+import subprocess+import time+import uuid+from dataclasses import asdict, dataclass+from pathlib import Path+from typing import Any++from smoke.lib.config import ProviderModel, SmokeConfig, redacted+from smoke.lib.server import RunningServer++REGRESSION_CLASSIFICATIONS = frozenset({"harness_bug", "product_failure"})++_HTTP_REGRESSION_PATTERNS = (+ r'POST /v1/messages[^"\n]* HTTP/1\.1" 4(?!01|03|04|08|09)\d\d',+ r'POST /v1/messages[^"\n]* HTTP/1\.1" 5\d\d',+)+_UPSTREAM_UNAVAILABLE_MARKERS = (+ "upstream_unavailable",+ "readtimeout",+ "connecterror",+ "connection refused",+ "timed out",+ "rate limit",+ "overloaded",+ "capacity",+ "upstream provider",+)+_HTTP_429_PATTERNS = (+ r'HTTP/1\.[01]" 429\b',+ r"\bHTTP/1\.[01] 429\b",+ r"\bstatus_code=429\b",+ r"\bstatus[=:]\s*429\b",+ r"\b429 Too Many Requests\b",+)+_MISSING_ENV_MARKERS = (+ "api key",+ "not logged in",+ "authentication",+ "permission denied",+)+_EMPTY_MCP_CONFIG = '{"mcpServers":{}}'+_SUBAGENT_SYSTEM_PROMPT = (+ "You are a deterministic smoke-test coordinator. Use Agent when asked to "+ "use a subagent."+)+++@dataclass(frozen=True, slots=True)+class ClaudeCliRun:+ command: tuple[str, ...]+ returncode: int | None+ stdout: str+ stderr: str+ duration_s: float+ timed_out: bool = False++ @property+ def combined_output(self) -> str:+ return f"{self.stdout}\n{self.stderr}"+++@dataclass(frozen=True, slots=True)+class CliMatrixOutcome:+ model: str+ full_model: str+ source: str+ feature: str+ outcome: str+ classification: str+ duration_s: float+ cli_returncode: int | None+ token_evidence: dict[str, Any]+ request_count: int+ log_path: str+ stdout_excerpt: str+ stderr_excerpt: str+ log_excerpt: str+++def run_claude_cli(+ *,+ claude_bin: str,+ server: RunningServer,+ config: SmokeConfig,+ cwd: Path,+ prompt: str,+ tools: str | None,+ bare: bool = True,+ pre_tool_args: tuple[str, ...] = (),+ extra_args: tuple[str, ...] = (),+ session_id: str | None = None,+ resume_session_id: str | None = None,+ no_session_persistence: bool = True,+) -> ClaudeCliRun:+ """Run Claude Code CLI against the local smoke proxy."""+ cwd.mkdir(parents=True, exist_ok=True)++ cmd = list(+ _build_claude_cli_command(+ claude_bin=claude_bin,+ prompt=prompt,+ tools=tools,+ bare=bare,+ pre_tool_args=pre_tool_args,+ extra_args=extra_args,+ session_id=session_id,+ resume_session_id=resume_session_id,+ no_session_persistence=no_session_persistence,+ )+ )++ env = os.environ.copy()+ env["ANTHROPIC_BASE_URL"] = server.base_url+ env["ANTHROPIC_API_URL"] = f"{server.base_url}/v1"+ env.setdefault("ANTHROPIC_API_KEY", "sk-smoke-proxy")+ if config.settings.anthropic_auth_token:+ env["ANTHROPIC_AUTH_TOKEN"] = config.settings.anthropic_auth_token+ env["TERM"] = "dumb"+ env["NO_COLOR"] = "1"+ env["PYTHONIOENCODING"] = "utf-8"++ started = time.monotonic()+ try:+ result = subprocess.run(+ cmd,+ cwd=cwd,+ env=env,+ capture_output=True,+ text=True,+ timeout=config.timeout_s,+ check=False,+ )+ except subprocess.TimeoutExpired as exc:+ return ClaudeCliRun(+ command=tuple(cmd),+ returncode=None,+ stdout=_coerce_timeout_text(exc.stdout),+ stderr=_coerce_timeout_text(exc.stderr),+ duration_s=time.monotonic() - started,+ timed_out=True,+ )++ return ClaudeCliRun(+ command=tuple(cmd),+ returncode=result.returncode,+ stdout=result.stdout,+ stderr=result.stderr,+ duration_s=time.monotonic() - started,+ )+++def _build_claude_cli_command(+ *,+ claude_bin: str,+ prompt: str,+ tools: str | None,+ bare: bool = True,+ pre_tool_args: tuple[str, ...] = (),+ extra_args: tuple[str, ...] = (),+ session_id: str | None = None,+ resume_session_id: str | None = None,+ no_session_persistence: bool = True,+) -> tuple[str, ...]:+ cmd: list[str] = [claude_bin]+ if bare:+ cmd.append("--bare")+ if resume_session_id:+ cmd.extend(["--resume", resume_session_id])+ if session_id:+ cmd.extend(["--session-id", session_id])+ cmd.extend(+ [+ "--output-format",+ "stream-json",+ "--include-partial-messages",+ "--verbose",+ "--permission-mode",+ "bypassPermissions",+ "--dangerously-skip-permissions",+ "--model",+ "sonnet",+ ]+ )+ if no_session_persistence:+ cmd.append("--no-session-persistence")+ cmd.extend(pre_tool_args)+ if tools is not None:+ cmd.extend(["--tools", tools])+ if tools:+ cmd.extend(["--allowedTools", tools])+ cmd.extend(extra_args)+ cmd.extend(["-p", prompt])+ return tuple(cmd)+++def run_cli_feature_probes(+ *,+ claude_bin: str,+ server: RunningServer,+ smoke_config: SmokeConfig,+ provider_model: ProviderModel,+ model_dir: Path,+ marker_prefix: str,+) -> list[CliMatrixOutcome]:+ return [+ _basic_text(+ claude_bin, server, smoke_config, provider_model, model_dir, marker_prefix+ ),+ _thinking(+ claude_bin, server, smoke_config, provider_model, model_dir, marker_prefix+ ),+ _tool_use_roundtrip(+ claude_bin, server, smoke_config, provider_model, model_dir, marker_prefix+ ),+ _interleaved_thinking_tool(+ claude_bin, server, smoke_config, provider_model, model_dir, marker_prefix+ ),+ _subagent_task(+ claude_bin, server, smoke_config, provider_model, model_dir, marker_prefix+ ),+ _compact_command(+ claude_bin, server, smoke_config, provider_model, model_dir, marker_prefix+ ),+ ]+++def read_log_offset(log_path: Path) -> int:+ """Return the current text length of a smoke server log."""+ if not log_path.is_file():+ return 0+ return len(log_path.read_text(encoding="utf-8", errors="replace"))+++def read_log_delta(log_path: Path, offset: int) -> str:+ """Return smoke server log text written after ``offset``."""+ if not log_path.is_file():+ return ""+ text = log_path.read_text(encoding="utf-8", errors="replace")+ return text[offset:]+++def token_evidence(+ *,+ feature: str,+ marker: str,+ run: ClaudeCliRun,+ log_delta: str,+) -> dict[str, Any]:+ """Collect compact evidence for a CLI feature probe."""+ combined = f"{run.combined_output}\n{log_delta}"+ lower = combined.lower()+ return {+ "feature": feature,+ "marker_present": bool(marker and marker in combined),+ "thinking_delta_count": combined.count("thinking_delta"),+ "tool_use_count": combined.count('"tool_use"'),+ "tool_result_count": combined.count('"tool_result"'),+ "agent_catalog_present": _tool_catalog_has(log_delta, "Agent"),+ "agent_tool_count": _agent_tool_count(combined),+ "agent_result_count": _agent_result_count(combined),+ "task_tool_count": combined.count('"name": "Task"')+ + combined.count('"name":"Task"'),+ "run_in_background_false": "run_in_background" in combined and "false" in lower,+ "compact_boundary": "compact_boundary" in combined,+ "compact_metadata": "compact_metadata" in combined,+ "http_422": 'HTTP/1.1" 422' in combined,+ "http_500": bool(re.search(r'HTTP/1\.1" 5\d\d', combined)),+ "timed_out": run.timed_out,+ }+++def classify_probe(+ *,+ run: ClaudeCliRun,+ log_delta: str,+ marker: str,+ requires_tool_result: bool = False,+ requires_agent: bool = False,+ requires_task: bool = False,+ requires_compact: bool = False,+) -> tuple[str, str]:+ """Classify a probe without failing compatibility characterization failures."""+ combined = f"{run.combined_output}\n{log_delta}"+ lower = combined.lower()++ if _has_proxy_regression(log_delta):+ return "failed", "product_failure"+ if run.returncode != 0 and any(+ marker_text in lower for marker_text in _MISSING_ENV_MARKERS+ ):+ return "skipped", "missing_env"+ if run.timed_out:+ return "failed", "probe_timeout"+ if requires_agent and not _tool_catalog_has(log_delta, "Agent"):+ return "failed", "harness_bug"++ marker_ok = not marker or marker in combined+ tool_ok = not requires_tool_result or '"tool_result"' in combined+ agent_ok = not requires_agent or (+ _agent_tool_count(combined) > 0 and _agent_result_count(combined) > 0+ )+ task_ok = not requires_task or (+ ('"name": "Task"' in combined or '"name":"Task"' in combined)+ and "run_in_background" in combined+ and "false" in lower+ )+ compact_ok = not requires_compact or (+ "compact_boundary" in combined+ or "compact_metadata" in combined+ or "/compact" in combined+ or "compact" in lower+ )+ cli_ok = run.returncode == 0++ if cli_ok and marker_ok and tool_ok and agent_ok and task_ok and compact_ok:+ return "passed", "passed"+ if _has_upstream_unavailable_text(combined):+ return "failed", "upstream_unavailable"+ if not _has_proxy_request(log_delta):+ return "failed", "harness_bug"+ return "failed", "model_feature_failure"+++def make_outcome(+ *,+ model: str,+ full_model: str,+ source: str,+ feature: str,+ marker: str,+ run: ClaudeCliRun,+ log_delta: str,+ log_path: Path,+ requires_tool_result: bool = False,+ requires_agent: bool = False,+ requires_task: bool = False,+ requires_compact: bool = False,+) -> CliMatrixOutcome:+ """Build one report outcome from a CLI run and its server log delta."""+ outcome, classification = classify_probe(+ run=run,+ log_delta=log_delta,+ marker=marker,+ requires_tool_result=requires_tool_result,+ requires_agent=requires_agent,+ requires_task=requires_task,+ requires_compact=requires_compact,+ )+ evidence = token_evidence(+ feature=feature,+ marker=marker,+ run=run,+ log_delta=log_delta,+ )+ return CliMatrixOutcome(+ model=model,+ full_model=full_model,+ source=source,+ feature=feature,+ outcome=outcome,+ classification=classification,+ duration_s=round(run.duration_s, 3),+ cli_returncode=run.returncode,+ token_evidence=evidence,+ request_count=_request_count(log_delta),+ log_path=str(log_path),+ stdout_excerpt=_excerpt(run.stdout),+ stderr_excerpt=_excerpt(run.stderr),+ log_excerpt=_excerpt(log_delta),+ )+++def write_matrix_report(+ config: SmokeConfig,+ outcomes: list[CliMatrixOutcome],+ *,+ target: str,+ filename_prefix: str,+) -> Path:+ """Write a Claude CLI compatibility matrix report."""+ config.results_dir.mkdir(parents=True, exist_ok=True)+ path = (+ config.results_dir+ / f"{filename_prefix}-matrix-{config.worker_id}-{int(time.time())}.json"+ )+ payload = {+ "started_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),+ "worker_id": config.worker_id,+ "target": target,+ "models": sorted({outcome.full_model for outcome in outcomes}),+ "outcomes": [asdict(outcome) for outcome in outcomes],+ }+ path.write_text(json.dumps(payload, indent=2, sort_keys=True), encoding="utf-8")+ return path+++def regression_failures(outcomes: list[CliMatrixOutcome]) -> list[str]:+ """Return report lines for classifications that should fail pytest."""+ return [+ f"{outcome.full_model} {outcome.feature}: {outcome.classification}"+ for outcome in outcomes+ if outcome.classification in REGRESSION_CLASSIFICATIONS+ ]+++def _basic_text(+ claude_bin: str,+ server: RunningServer,+ smoke_config: SmokeConfig,+ provider_model: ProviderModel,+ model_dir: Path,+ marker_prefix: str,+) -> CliMatrixOutcome:+ marker = _marker(marker_prefix, "BASIC")+ return _run_probe(+ claude_bin=claude_bin,+ server=server,+ smoke_config=smoke_config,+ provider_model=provider_model,+ workspace=model_dir / "basic_text",+ feature="basic_text",+ marker=marker,+ prompt=f"Reply with exactly {marker} and no other text.",+ tools="",+ )+++def _thinking(+ claude_bin: str,+ server: RunningServer,+ smoke_config: SmokeConfig,+ provider_model: ProviderModel,+ model_dir: Path,+ marker_prefix: str,+) -> CliMatrixOutcome:+ marker = _marker(marker_prefix, "THINK")+ return _run_probe(+ claude_bin=claude_bin,+ server=server,+ smoke_config=smoke_config,+ provider_model=provider_model,+ workspace=model_dir / "thinking",+ feature="thinking",+ marker=marker,+ prompt=(+ "Think privately about the request, then reply with exactly "+ f"{marker} and no other text."+ ),+ tools="",+ extra_args=("--effort", "high"),+ )+++def _tool_use_roundtrip(+ claude_bin: str,+ server: RunningServer,+ smoke_config: SmokeConfig,+ provider_model: ProviderModel,+ model_dir: Path,+ marker_prefix: str,+) -> CliMatrixOutcome:+ marker = _marker(marker_prefix, "TOOL")+ workspace = model_dir / "tool_use_roundtrip"+ (workspace / "smoke-read.txt").parent.mkdir(parents=True, exist_ok=True)+ (workspace / "smoke-read.txt").write_text(marker, encoding="utf-8")+ return _run_probe(+ claude_bin=claude_bin,+ server=server,+ smoke_config=smoke_config,+ provider_model=provider_model,+ workspace=workspace,+ feature="tool_use_roundtrip",+ marker=marker,+ prompt=(+ "Use the Read tool to read smoke-read.txt. Reply with exactly the "+ "secret token from that file and no other text."+ ),+ tools="Read",+ requires_tool_result=True,+ )+++def _interleaved_thinking_tool(+ claude_bin: str,+ server: RunningServer,+ smoke_config: SmokeConfig,+ provider_model: ProviderModel,+ model_dir: Path,+ marker_prefix: str,+) -> CliMatrixOutcome:+ marker = _marker(marker_prefix, "INTERLEAVED")+ workspace = model_dir / "interleaved_thinking_tool"+ (workspace / "smoke-interleaved.txt").parent.mkdir(parents=True, exist_ok=True)+ (workspace / "smoke-interleaved.txt").write_text(marker, encoding="utf-8")+ return _run_probe(+ claude_bin=claude_bin,+ server=server,+ smoke_config=smoke_config,+ provider_model=provider_model,+ workspace=workspace,+ feature="interleaved_thinking_tool",+ marker=marker,+ prompt=(+ "Think privately, use Read on smoke-interleaved.txt, then reply with "+ "exactly the secret token from that file and no other text."+ ),+ tools="Read",+ extra_args=("--effort", "high"),+ requires_tool_result=True,+ )+++def _subagent_task(+ claude_bin: str,+ server: RunningServer,+ smoke_config: SmokeConfig,+ provider_model: ProviderModel,+ model_dir: Path,+ marker_prefix: str,+) -> CliMatrixOutcome:+ marker = _marker(marker_prefix, "TASK")+ workspace = model_dir / "subagent_task"+ (workspace / "smoke-subagent.txt").parent.mkdir(parents=True, exist_ok=True)+ (workspace / "smoke-subagent.txt").write_text(marker, encoding="utf-8")+ agents = json.dumps(+ {+ "smoke_reader": {+ "description": "Reads one requested file and returns its token.",+ "prompt": (+ "Read the requested file with Read and return only the token "+ "inside it."+ ),+ "tools": ["Read"],+ "permissionMode": "bypassPermissions",+ "background": False,+ }+ }+ )+ bare, tools, pre_tool_args, extra_args = _subagent_probe_options(agents)+ return _run_probe(+ claude_bin=claude_bin,+ server=server,+ smoke_config=smoke_config,+ provider_model=provider_model,+ workspace=workspace,+ feature="subagent_task",+ marker=marker,+ prompt=(+ "Use the smoke_reader subagent to read smoke-subagent.txt. After the "+ "first agent result, reply with exactly the token and stop. Do not "+ "call any other tools."+ ),+ tools=tools,+ bare=bare,+ pre_tool_args=pre_tool_args,+ extra_args=extra_args,+ requires_tool_result=True,+ requires_agent=True,+ )+++def _subagent_probe_options(+ agents: str,+) -> tuple[bool, str, tuple[str, ...], tuple[str, ...]]:+ return (+ False,+ "Agent,Read",+ (+ "--setting-sources",+ "local",+ "--strict-mcp-config",+ "--mcp-config",+ _EMPTY_MCP_CONFIG,+ "--system-prompt",+ _SUBAGENT_SYSTEM_PROMPT,+ ),+ ("--agents", agents),+ )+++def _compact_command(+ claude_bin: str,+ server: RunningServer,+ smoke_config: SmokeConfig,+ provider_model: ProviderModel,+ model_dir: Path,+ marker_prefix: str,+) -> CliMatrixOutcome:+ marker = _marker(marker_prefix, "COMPACT")+ workspace = model_dir / "compact_command"+ session_id = str(uuid.uuid4())+ offset = read_log_offset(server.log_path)+ first = run_claude_cli(+ claude_bin=claude_bin,+ server=server,+ config=smoke_config,+ cwd=workspace,+ prompt=f"Remember this smoke token: {marker}. Reply with exactly {marker}.",+ tools="",+ session_id=session_id,+ no_session_persistence=False,+ )+ second = run_claude_cli(+ claude_bin=claude_bin,+ server=server,+ config=smoke_config,+ cwd=workspace,+ prompt=f"/compact preserve {marker}",+ tools="",+ resume_session_id=session_id,+ no_session_persistence=False,+ )+ log_delta = read_log_delta(server.log_path, offset)+ run = ClaudeCliRun(+ command=(*first.command, "&&", *second.command),+ returncode=second.returncode if first.returncode == 0 else first.returncode,+ stdout=f"{first.stdout}\n{second.stdout}",+ stderr=f"{first.stderr}\n{second.stderr}",+ duration_s=first.duration_s + second.duration_s,+ timed_out=first.timed_out or second.timed_out,+ )+ return make_outcome(+ model=provider_model.model_name,+ full_model=provider_model.full_model,+ source=provider_model.source,+ feature="compact_command",+ marker="",+ run=run,+ log_delta=log_delta,+ log_path=server.log_path,+ requires_compact=True,+ )+++def _run_probe(+ *,+ claude_bin: str,+ server: RunningServer,+ smoke_config: SmokeConfig,+ provider_model: ProviderModel,+ workspace: Path,+ feature: str,+ marker: str,+ prompt: str,+ tools: str | None,+ bare: bool = True,+ pre_tool_args: tuple[str, ...] = (),+ extra_args: tuple[str, ...] = (),+ requires_tool_result: bool = False,+ requires_agent: bool = False,+ requires_task: bool = False,+) -> CliMatrixOutcome:+ offset = read_log_offset(server.log_path)+ run = run_claude_cli(+ claude_bin=claude_bin,+ server=server,+ config=smoke_config,+ cwd=workspace,+ prompt=prompt,+ tools=tools,+ bare=bare,+ pre_tool_args=pre_tool_args,+ extra_args=extra_args,+ )+ log_delta = read_log_delta(server.log_path, offset)+ return make_outcome(+ model=provider_model.model_name,+ full_model=provider_model.full_model,+ source=provider_model.source,+ feature=feature,+ marker=marker,+ run=run,+ log_delta=log_delta,+ log_path=server.log_path,+ requires_tool_result=requires_tool_result,+ requires_agent=requires_agent,+ requires_task=requires_task,+ )+++def _has_proxy_regression(log_delta: str) -> bool:+ if "CREATE_MESSAGE_ERROR" in log_delta:+ return True+ return any(re.search(pattern, log_delta) for pattern in _HTTP_REGRESSION_PATTERNS)+++def _has_proxy_request(log_delta: str) -> bool:+ return "POST /v1/messages" in log_delta or "API_REQUEST:" in log_delta+++def _tool_catalog_has(log_delta: str, tool_name: str) -> bool:+ catalog = _first_tool_catalog(log_delta)+ return (+ f"'name': '{tool_name}'" in catalog+ or f'"name": "{tool_name}"' in catalog+ or f'"name":"{tool_name}"' in catalog+ )+++def _first_tool_catalog(log_delta: str) -> str:+ for line in log_delta.splitlines():+ if "FULL_PAYLOAD" not in line:+ continue+ single_index = line.find("'tools': [")+ double_index = line.find('"tools": [')+ if single_index == -1 and double_index == -1:+ continue+ start = single_index if single_index != -1 else double_index+ end_candidates = [+ index+ for marker in ("'tool_choice'", '"tool_choice"', "'thinking'", '"thinking"')+ if (index := line.find(marker, start)) != -1+ ]+ end = min(end_candidates) if end_candidates else len(line)+ return line[start:end]+ return ""+++def _agent_tool_count(text: str) -> int:+ return (+ text.count('"name": "Agent"')+ + text.count('"name":"Agent"')+ + len(+ re.findall(+ r"'type': 'tool_use'[^}\n]+?'name': 'Agent'",+ text,+ flags=re.DOTALL,+ )+ )+ )+++def _agent_result_count(text: str) -> int:+ return text.count("agentId:") + text.count('"agentId"') + text.count("'agentId'")+++def _has_upstream_unavailable_text(text: str) -> bool:+ lower = text.lower()+ if any(marker_text in lower for marker_text in _UPSTREAM_UNAVAILABLE_MARKERS):+ return True+ return any(+ re.search(pattern, text, flags=re.IGNORECASE) for pattern in _HTTP_429_PATTERNS+ )+++def _request_count(log_delta: str) -> int:+ access_log_count = log_delta.count("POST /v1/messages")+ service_log_count = log_delta.count("API_REQUEST:")+ return max(access_log_count, service_log_count)+++def _marker(scope: str, prefix: str) -> str:+ return f"FCC_{scope}_{prefix}_{uuid.uuid4().hex[:8].upper()}"+++def _excerpt(value: str, *, max_chars: int = 2400) -> str:+ if len(value) <= max_chars:+ return redacted(value)+ return redacted(value[-max_chars:])+++def _coerce_timeout_text(value: str | bytes | None) -> str:+ if value is None:+ return ""+ if isinstance(value, bytes):+ return value.decode("utf-8", errors="replace")+ return valuediff --git a/smoke/lib/config.py b/smoke/lib/config.pyindex a5947cb0..f23f28c0 100644--- a/smoke/lib/config.py+++ b/smoke/lib/config.py@@ -28,11 +28,13 @@ DEFAULT_TARGETS = frozenset(})SIDE_EFFECT_TARGETS = frozenset({"discord", "telegram", "voice"})-OPT_IN_TARGETS = frozenset({"nvidia_nim_cli"})+OPT_IN_TARGETS = frozenset({"nvidia_nim_cli", "openrouter_free_cli"})ALL_TARGETS = DEFAULT_TARGETS | SIDE_EFFECT_TARGETS | OPT_IN_TARGETSTARGET_ALIASES = {"contract": "api","nim_cli": "nvidia_nim_cli",+ "openrouter_cli": "openrouter_free_cli",+ "openrouter_free": "openrouter_free_cli","optimizations": "api","thinking": "providers","vscode": "clients",@@ -58,6 +60,14 @@ NVIDIA_NIM_CLI_DEFAULT_MODELS: tuple[str, ...] = ("deepseek-ai/deepseek-v4-flash",)+OPENROUTER_FREE_CLI_DEFAULT_MODELS: tuple[str, ...] = (+ "nvidia/nemotron-3-super-120b-a12b:free",+ "openai/gpt-oss-120b:free",+ "minimax/minimax-m2.5:free",+ "inclusionai/ring-2.6-1t:free",+ "poolside/laguna-m.1:free",+)+TARGET_REQUIRED_ENV: dict[str, tuple[str, ...]] = {"api": (),@@ -77,6 +87,10 @@ TARGET_REQUIRED_ENV: dict[str, tuple[str, ...]] = {"NVIDIA_NIM_API_KEY","FCC_SMOKE_CLAUDE_BIN or claude on PATH",),+ "openrouter_free_cli": (+ "OPENROUTER_API_KEY",+ "FCC_SMOKE_CLAUDE_BIN or claude on PATH",+ ),"telegram": ("TELEGRAM_BOT_TOKEN","ALLOWED_TELEGRAM_USER_ID or FCC_SMOKE_TELEGRAM_CHAT_ID",@@ -183,6 +197,13 @@ class SmokeConfig:for full_model, source in nvidia_nim_cli_model_refs().items()]+ def openrouter_free_cli_models(self) -> list[ProviderModel]:+ """Return OpenRouter free models for Claude Code CLI characterization."""+ return [+ ProviderModel(provider="open_router", full_model=full_model, source=source)+ for full_model, source in openrouter_free_cli_model_refs().items()+ ]+def _include_provider_in_smoke(self, provider: str, mapped_providers: set[str]) -> bool:@@ -295,6 +316,40 @@ def nvidia_nim_cli_model_refs(return normalized+def openrouter_free_cli_model_refs(+ env: Mapping[str, str] | None = None,+) -> dict[str, str]:+ """Return normalized OpenRouter free CLI matrix model refs in deterministic order."""+ source = env if env is not None else os.environ+ explicit_models = _parse_csv_ordered(source.get("FCC_SMOKE_OPENROUTER_FREE_MODELS"))+ extra_models = _parse_csv_ordered(+ source.get("FCC_SMOKE_OPENROUTER_FREE_EXTRA_MODELS")+ )++ if "FCC_SMOKE_OPENROUTER_FREE_MODELS" in source and not explicit_models:+ raise ValueError(+ "FCC_SMOKE_OPENROUTER_FREE_MODELS must list at least one model"+ )++ models: list[tuple[str, str]] = []+ base_models = explicit_models or OPENROUTER_FREE_CLI_DEFAULT_MODELS+ base_source = (+ "FCC_SMOKE_OPENROUTER_FREE_MODELS"+ if explicit_models+ else "openrouter_free_cli_default"+ )+ models.extend((model, base_source) for model in base_models)+ models.extend(+ (model, "FCC_SMOKE_OPENROUTER_FREE_EXTRA_MODELS") for model in extra_models+ )++ normalized: dict[str, str] = {}+ for raw_model, model_source in models:+ full_model = _normalize_provider_model("open_router", raw_model)+ normalized.setdefault(full_model, model_source)+ return normalized++def auth_headers(token: str | None = None) -> dict[str, str]:settings = get_settings()resolved = token if token is not None else settings.anthropic_auth_tokendiff --git a/smoke/lib/nvidia_nim_cli.py b/smoke/lib/nvidia_nim_cli.pydeleted file mode 100644index 30a85a54..00000000--- a/smoke/lib/nvidia_nim_cli.py+++ /dev/null@@ -1,350 +0,0 @@-"""Claude Code CLI characterization helpers for NVIDIA NIM smoke tests."""--from __future__ import annotations--import json-import os-import re-import subprocess-import time-from dataclasses import asdict, dataclass-from pathlib import Path-from typing import Any--from smoke.lib.config import SmokeConfig, redacted-from smoke.lib.server import RunningServer--REGRESSION_CLASSIFICATIONS = frozenset({"harness_bug", "product_failure"})--_HTTP_REGRESSION_PATTERNS = (- r'POST /v1/messages[^"\n]* HTTP/1\.1" 4(?!01|03|04|08|09)\d\d',- r'POST /v1/messages[^"\n]* HTTP/1\.1" 5\d\d',-)-_UPSTREAM_UNAVAILABLE_MARKERS = (- "upstream_unavailable",- "readtimeout",- "connecterror",- "connection refused",- "timed out",- "rate limit",- "429",- "overloaded",- "capacity",- "upstream provider",-)-_MISSING_ENV_MARKERS = (- "api key",- "not logged in",- "authentication",- "permission denied",-)---@dataclass(frozen=True, slots=True)-class ClaudeCliRun:- command: tuple[str, ...]- returncode: int | None- stdout: str- stderr: str- duration_s: float- timed_out: bool = False-- @property- def combined_output(self) -> str:- return f"{self.stdout}\n{self.stderr}"---@dataclass(frozen=True, slots=True)-class NimCliMatrixOutcome:- model: str- full_model: str- source: str- feature: str- outcome: str- classification: str- duration_s: float- cli_returncode: int | None- token_evidence: dict[str, Any]- request_count: int- log_path: str- stdout_excerpt: str- stderr_excerpt: str- log_excerpt: str---def run_claude_cli(- *,- claude_bin: str,- server: RunningServer,- config: SmokeConfig,- cwd: Path,- prompt: str,- tools: str | None,- extra_args: tuple[str, ...] = (),- session_id: str | None = None,- resume_session_id: str | None = None,- no_session_persistence: bool = True,-) -> ClaudeCliRun:- """Run Claude Code CLI against the local smoke proxy."""- cwd.mkdir(parents=True, exist_ok=True)-- cmd: list[str] = [claude_bin, "--bare"]- if resume_session_id:- cmd.extend(["--resume", resume_session_id])- if session_id:- cmd.extend(["--session-id", session_id])- cmd.extend(- [- "--output-format",- "stream-json",- "--include-partial-messages",- "--verbose",- "--permission-mode",- "bypassPermissions",- "--dangerously-skip-permissions",- "--model",- "sonnet",- ]- )- if no_session_persistence:- cmd.append("--no-session-persistence")- if tools is not None:- cmd.extend(["--tools", tools])- if tools:- cmd.extend(["--allowedTools", tools])- cmd.extend(extra_args)- cmd.extend(["-p", prompt])-- env = os.environ.copy()- env["ANTHROPIC_BASE_URL"] = server.base_url- env["ANTHROPIC_API_URL"] = f"{server.base_url}/v1"- env.setdefault("ANTHROPIC_API_KEY", "sk-smoke-proxy")- if config.settings.anthropic_auth_token:- env["ANTHROPIC_AUTH_TOKEN"] = config.settings.anthropic_auth_token- env["TERM"] = "dumb"- env["NO_COLOR"] = "1"- env["PYTHONIOENCODING"] = "utf-8"-- started = time.monotonic()- try:- result = subprocess.run(- cmd,- cwd=cwd,- env=env,- capture_output=True,- text=True,- timeout=config.timeout_s,- check=False,- )- except subprocess.TimeoutExpired as exc:- return ClaudeCliRun(- command=tuple(cmd),- returncode=None,- stdout=_coerce_timeout_text(exc.stdout),- stderr=_coerce_timeout_text(exc.stderr),- duration_s=time.monotonic() - started,- timed_out=True,- )-- return ClaudeCliRun(- command=tuple(cmd),- returncode=result.returncode,- stdout=result.stdout,- stderr=result.stderr,- duration_s=time.monotonic() - started,- )---def read_log_offset(log_path: Path) -> int:- """Return the current text length of a smoke server log."""- if not log_path.is_file():- return 0- return len(log_path.read_text(encoding="utf-8", errors="replace"))---def read_log_delta(log_path: Path, offset: int) -> str:- """Return smoke server log text written after ``offset``."""- if not log_path.is_file():- return ""- text = log_path.read_text(encoding="utf-8", errors="replace")- return text[offset:]---def token_evidence(- *,- feature: str,- marker: str,- run: ClaudeCliRun,- log_delta: str,-) -> dict[str, Any]:- """Collect compact evidence for a CLI feature probe."""- combined = f"{run.combined_output}\n{log_delta}"- lower = combined.lower()- return {- "feature": feature,- "marker_present": bool(marker and marker in combined),- "thinking_delta_count": combined.count("thinking_delta"),- "tool_use_count": combined.count('"tool_use"'),- "tool_result_count": combined.count('"tool_result"'),- "task_tool_count": combined.count('"name": "Task"')- + combined.count('"name":"Task"'),- "run_in_background_false": "run_in_background" in combined and "false" in lower,- "compact_boundary": "compact_boundary" in combined,- "compact_metadata": "compact_metadata" in combined,- "http_422": 'HTTP/1.1" 422' in combined,- "http_500": bool(re.search(r'HTTP/1\.1" 5\d\d', combined)),- "timed_out": run.timed_out,- }---def classify_probe(- *,- run: ClaudeCliRun,- log_delta: str,- marker: str,- requires_tool_result: bool = False,- requires_task: bool = False,- requires_compact: bool = False,-) -> tuple[str, str]:- """Classify a probe without failing compatibility characterization failures."""- combined = f"{run.combined_output}\n{log_delta}"- lower = combined.lower()-- if _has_proxy_regression(log_delta):- return "failed", "product_failure"- if run.returncode != 0 and any(- marker_text in lower for marker_text in _MISSING_ENV_MARKERS- ):- return "skipped", "missing_env"- if run.timed_out:- return "failed", "probe_timeout"-- marker_ok = not marker or marker in combined- tool_ok = not requires_tool_result or '"tool_result"' in combined- task_ok = not requires_task or (- ('"name": "Task"' in combined or '"name":"Task"' in combined)- and "run_in_background" in combined- and "false" in lower- )- compact_ok = not requires_compact or (- "compact_boundary" in combined- or "compact_metadata" in combined- or "/compact" in combined- or "compact" in lower- )- cli_ok = run.returncode == 0-- if cli_ok and marker_ok and tool_ok and task_ok and compact_ok:- return "passed", "passed"- if any(marker_text in lower for marker_text in _UPSTREAM_UNAVAILABLE_MARKERS):- return "failed", "upstream_unavailable"- if not _has_proxy_request(log_delta):- return "failed", "harness_bug"- return "failed", "model_feature_failure"---def make_outcome(- *,- model: str,- full_model: str,- source: str,- feature: str,- marker: str,- run: ClaudeCliRun,- log_delta: str,- log_path: Path,- requires_tool_result: bool = False,- requires_task: bool = False,- requires_compact: bool = False,-) -> NimCliMatrixOutcome:- """Build one report outcome from a CLI run and its server log delta."""- outcome, classification = classify_probe(- run=run,- log_delta=log_delta,- marker=marker,- requires_tool_result=requires_tool_result,- requires_task=requires_task,- requires_compact=requires_compact,- )- evidence = token_evidence(- feature=feature,- marker=marker,- run=run,- log_delta=log_delta,- )- return NimCliMatrixOutcome(- model=model,- full_model=full_model,- source=source,- feature=feature,- outcome=outcome,- classification=classification,- duration_s=round(run.duration_s, 3),- cli_returncode=run.returncode,- token_evidence=evidence,- request_count=_request_count(log_delta),- log_path=str(log_path),- stdout_excerpt=_excerpt(run.stdout),- stderr_excerpt=_excerpt(run.stderr),- log_excerpt=_excerpt(log_delta),- )---def write_matrix_report(- config: SmokeConfig,- outcomes: list[NimCliMatrixOutcome],-) -> Path:- """Write the NVIDIA NIM CLI compatibility matrix report."""- config.results_dir.mkdir(parents=True, exist_ok=True)- path = (- config.results_dir- / f"nvidia-nim-cli-matrix-{config.worker_id}-{int(time.time())}.json"- )- payload = {- "started_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),- "worker_id": config.worker_id,- "target": "nvidia_nim_cli",- "models": sorted({outcome.full_model for outcome in outcomes}),- "outcomes": [asdict(outcome) for outcome in outcomes],- }- path.write_text(json.dumps(payload, indent=2, sort_keys=True), encoding="utf-8")- return path---def regression_failures(outcomes: list[NimCliMatrixOutcome]) -> list[str]:- """Return report lines for classifications that should fail pytest."""- return [- f"{outcome.full_model} {outcome.feature}: {outcome.classification}"- for outcome in outcomes- if outcome.classification in REGRESSION_CLASSIFICATIONS- ]---def _has_proxy_regression(log_delta: str) -> bool:- if "CREATE_MESSAGE_ERROR" in log_delta:- return True- return any(re.search(pattern, log_delta) for pattern in _HTTP_REGRESSION_PATTERNS)---def _has_proxy_request(log_delta: str) -> bool:- return "POST /v1/messages" in log_delta or "API_REQUEST:" in log_delta---def _request_count(log_delta: str) -> int:- access_log_count = log_delta.count("POST /v1/messages")- service_log_count = log_delta.count("API_REQUEST:")- return max(access_log_count, service_log_count)---def _excerpt(value: str, *, max_chars: int = 2400) -> str:- if len(value) <= max_chars:- return redacted(value)- return redacted(value[-max_chars:])---def _coerce_timeout_text(value: str | bytes | None) -> str:- if value is None:- return ""- if isinstance(value, bytes):- return value.decode("utf-8", errors="replace")- return valuediff --git a/smoke/product/test_nvidia_nim_cli_product_live.py b/smoke/product/test_nvidia_nim_cli_product_live.pyindex f7e3e391..674db1be 100644--- a/smoke/product/test_nvidia_nim_cli_product_live.py+++ b/smoke/product/test_nvidia_nim_cli_product_live.py@@ -1,25 +1,18 @@from __future__ import annotations-import jsonimport shutil-import uuidfrom pathlib import Pathimport pytest-from smoke.lib.config import ProviderModel, SmokeConfig-from smoke.lib.e2e import SmokeServerDriver-from smoke.lib.nvidia_nim_cli import (- ClaudeCliRun,- NimCliMatrixOutcome,- make_outcome,- read_log_delta,- read_log_offset,+from smoke.lib.claude_cli_matrix import (+ CliMatrixOutcome,regression_failures,- run_claude_cli,+ run_cli_feature_probes,write_matrix_report,)-from smoke.lib.server import RunningServer+from smoke.lib.config import SmokeConfig+from smoke.lib.e2e import SmokeServerDriverpytestmark = [pytest.mark.live, pytest.mark.smoke_target("nvidia_nim_cli")]@@ -36,7 +29,7 @@ def test_nvidia_nim_cli_matrix_e2e(smoke_config: SmokeConfig, tmp_path: Path) ->if not provider_models:pytest.skip("missing_env: no NVIDIA NIM CLI smoke models configured")- outcomes: list[NimCliMatrixOutcome] = []+ outcomes: list[CliMatrixOutcome] = []for provider_model in provider_models:with SmokeServerDriver(smoke_config,@@ -49,31 +42,23 @@ def test_nvidia_nim_cli_matrix_e2e(smoke_config: SmokeConfig, tmp_path: Path) ->"LOG_RAW_SSE_EVENTS": "true",},).run() as server:- model_dir = tmp_path / _slug(provider_model.model_name)outcomes.extend(- [- _basic_text(- claude_bin, server, smoke_config, provider_model, model_dir- ),- _thinking(- claude_bin, server, smoke_config, provider_model, model_dir- ),- _tool_use_roundtrip(- claude_bin, server, smoke_config, provider_model, model_dir- ),- _interleaved_thinking_tool(- claude_bin, server, smoke_config, provider_model, model_dir- ),- _subagent_task(- claude_bin, server, smoke_config, provider_model, model_dir- ),- _compact_command(- claude_bin, server, smoke_config, provider_model, model_dir- ),- ]+ run_cli_feature_probes(+ claude_bin=claude_bin,+ server=server,+ smoke_config=smoke_config,+ provider_model=provider_model,+ model_dir=tmp_path / _slug(provider_model.model_name),+ marker_prefix="NIM",+ ))- report_path = write_matrix_report(smoke_config, outcomes)+ report_path = write_matrix_report(+ smoke_config,+ outcomes,+ target="nvidia_nim_cli",+ filename_prefix="nvidia-nim-cli",+ )failures = regression_failures(outcomes)assert not failures, (f"NVIDIA NIM CLI matrix regressions written to {report_path}:\n"@@ -81,245 +66,5 @@ def test_nvidia_nim_cli_matrix_e2e(smoke_config: SmokeConfig, tmp_path: Path) ->)-def _basic_text(- claude_bin: str,- server: RunningServer,- smoke_config: SmokeConfig,- provider_model: ProviderModel,- model_dir: Path,-) -> NimCliMatrixOutcome:- marker = _marker("BASIC")- return _run_probe(- claude_bin=claude_bin,- server=server,- smoke_config=smoke_config,- provider_model=provider_model,- workspace=model_dir / "basic_text",- feature="basic_text",- marker=marker,- prompt=f"Reply with exactly {marker} and no other text.",- tools="",- )---def _thinking(- claude_bin: str,- server: RunningServer,- smoke_config: SmokeConfig,- provider_model: ProviderModel,- model_dir: Path,-) -> NimCliMatrixOutcome:- marker = _marker("THINK")- return _run_probe(- claude_bin=claude_bin,- server=server,- smoke_config=smoke_config,- provider_model=provider_model,- workspace=model_dir / "thinking",- feature="thinking",- marker=marker,- prompt=(- "Think privately about the request, then reply with exactly "- f"{marker} and no other text."- ),- tools="",- extra_args=("--effort", "high"),- )---def _tool_use_roundtrip(- claude_bin: str,- server: RunningServer,- smoke_config: SmokeConfig,- provider_model: ProviderModel,- model_dir: Path,-) -> NimCliMatrixOutcome:- marker = _marker("TOOL")- workspace = model_dir / "tool_use_roundtrip"- (workspace / "smoke-read.txt").parent.mkdir(parents=True, exist_ok=True)- (workspace / "smoke-read.txt").write_text(marker, encoding="utf-8")- return _run_probe(- claude_bin=claude_bin,- server=server,- smoke_config=smoke_config,- provider_model=provider_model,- workspace=workspace,- feature="tool_use_roundtrip",- marker=marker,- prompt=(- "Use the Read tool to read smoke-read.txt. Reply with exactly the "- "secret token from that file and no other text."- ),- tools="Read",- requires_tool_result=True,- )---def _interleaved_thinking_tool(- claude_bin: str,- server: RunningServer,- smoke_config: SmokeConfig,- provider_model: ProviderModel,- model_dir: Path,-) -> NimCliMatrixOutcome:- marker = _marker("INTERLEAVED")- workspace = model_dir / "interleaved_thinking_tool"- (workspace / "smoke-interleaved.txt").parent.mkdir(parents=True, exist_ok=True)- (workspace / "smoke-interleaved.txt").write_text(marker, encoding="utf-8")- return _run_probe(- claude_bin=claude_bin,- server=server,- smoke_config=smoke_config,- provider_model=provider_model,- workspace=workspace,- feature="interleaved_thinking_tool",- marker=marker,- prompt=(- "Think privately, use Read on smoke-interleaved.txt, then reply with "- "exactly the secret token from that file and no other text."- ),- tools="Read",- extra_args=("--effort", "high"),- requires_tool_result=True,- )---def _subagent_task(- claude_bin: str,- server: RunningServer,- smoke_config: SmokeConfig,- provider_model: ProviderModel,- model_dir: Path,-) -> NimCliMatrixOutcome:- marker = _marker("TASK")- workspace = model_dir / "subagent_task"- (workspace / "smoke-subagent.txt").parent.mkdir(parents=True, exist_ok=True)- (workspace / "smoke-subagent.txt").write_text(marker, encoding="utf-8")- agents = json.dumps(- {- "smoke_reader": {- "description": "Reads one requested file and returns its token.",- "prompt": (- "Read the requested file with Read and return only the token "- "inside it."- ),- }- }- )- return _run_probe(- claude_bin=claude_bin,- server=server,- smoke_config=smoke_config,- provider_model=provider_model,- workspace=workspace,- feature="subagent_task",- marker=marker,- prompt=(- "Use the smoke_reader subagent with Task to read smoke-subagent.txt. "- "Reply with exactly the token the subagent returns and no other text."- ),- tools="Task,Read",- extra_args=("--agents", agents),- requires_tool_result=True,- )---def _compact_command(- claude_bin: str,- server: RunningServer,- smoke_config: SmokeConfig,- provider_model: ProviderModel,- model_dir: Path,-) -> NimCliMatrixOutcome:- marker = _marker("COMPACT")- workspace = model_dir / "compact_command"- session_id = str(uuid.uuid4())- offset = read_log_offset(server.log_path)- first = run_claude_cli(- claude_bin=claude_bin,- server=server,- config=smoke_config,- cwd=workspace,- prompt=f"Remember this smoke token: {marker}. Reply with exactly {marker}.",- tools="",- session_id=session_id,- no_session_persistence=False,- )- second = run_claude_cli(- claude_bin=claude_bin,- server=server,- config=smoke_config,- cwd=workspace,- prompt=f"/compact preserve {marker}",- tools="",- resume_session_id=session_id,- no_session_persistence=False,- )- log_delta = read_log_delta(server.log_path, offset)- run = ClaudeCliRun(- command=(*first.command, "&&", *second.command),- returncode=second.returncode if first.returncode == 0 else first.returncode,- stdout=f"{first.stdout}\n{second.stdout}",- stderr=f"{first.stderr}\n{second.stderr}",- duration_s=first.duration_s + second.duration_s,- timed_out=first.timed_out or second.timed_out,- )- return make_outcome(- model=provider_model.model_name,- full_model=provider_model.full_model,- source=provider_model.source,- feature="compact_command",- marker="",- run=run,- log_delta=log_delta,- log_path=server.log_path,- requires_compact=True,- )---def _run_probe(- *,- claude_bin: str,- server: RunningServer,- smoke_config: SmokeConfig,- provider_model: ProviderModel,- workspace: Path,- feature: str,- marker: str,- prompt: str,- tools: str | None,- extra_args: tuple[str, ...] = (),- requires_tool_result: bool = False,- requires_task: bool = False,-) -> NimCliMatrixOutcome:- offset = read_log_offset(server.log_path)- run = run_claude_cli(- claude_bin=claude_bin,- server=server,- config=smoke_config,- cwd=workspace,- prompt=prompt,- tools=tools,- extra_args=extra_args,- )- log_delta = read_log_delta(server.log_path, offset)- return make_outcome(- model=provider_model.model_name,- full_model=provider_model.full_model,- source=provider_model.source,- feature=feature,- marker=marker,- run=run,- log_delta=log_delta,- log_path=server.log_path,- requires_tool_result=requires_tool_result,- requires_task=requires_task,- )---def _marker(prefix: str) -> str:- return f"FCC_NIM_{prefix}_{uuid.uuid4().hex[:8].upper()}"--def _slug(value: str) -> str:return "".join(char if char.isalnum() else "-" for char in value).strip("-")diff --git a/smoke/product/test_openrouter_free_cli_product_live.py b/smoke/product/test_openrouter_free_cli_product_live.pynew file mode 100644index 00000000..ac36b8d0--- /dev/null+++ b/smoke/product/test_openrouter_free_cli_product_live.py@@ -0,0 +1,72 @@+from __future__ import annotations++import shutil+from pathlib import Path++import pytest++from smoke.lib.claude_cli_matrix import (+ CliMatrixOutcome,+ regression_failures,+ run_cli_feature_probes,+ write_matrix_report,+)+from smoke.lib.config import SmokeConfig+from smoke.lib.e2e import SmokeServerDriver++pytestmark = [pytest.mark.live, pytest.mark.smoke_target("openrouter_free_cli")]+++def test_openrouter_free_cli_matrix_e2e(+ smoke_config: SmokeConfig, tmp_path: Path+) -> None:+ if not smoke_config.has_provider_configuration("open_router"):+ pytest.skip("missing_env: OPENROUTER_API_KEY is not configured")++ claude_bin = shutil.which(smoke_config.claude_bin)+ if not claude_bin:+ pytest.skip(f"missing_env: Claude CLI not found: {smoke_config.claude_bin}")++ provider_models = smoke_config.openrouter_free_cli_models()+ if not provider_models:+ pytest.skip("missing_env: no OpenRouter free CLI smoke models configured")++ outcomes: list[CliMatrixOutcome] = []+ for provider_model in provider_models:+ with SmokeServerDriver(+ smoke_config,+ name=f"product-openrouter-free-cli-{_slug(provider_model.model_name)}",+ env_overrides={+ "MODEL": provider_model.full_model,+ "MESSAGING_PLATFORM": "none",+ "ENABLE_MODEL_THINKING": "true",+ "LOG_RAW_API_PAYLOADS": "true",+ "LOG_RAW_SSE_EVENTS": "true",+ },+ ).run() as server:+ outcomes.extend(+ run_cli_feature_probes(+ claude_bin=claude_bin,+ server=server,+ smoke_config=smoke_config,+ provider_model=provider_model,+ model_dir=tmp_path / _slug(provider_model.model_name),+ marker_prefix="OPENROUTER_FREE",+ )+ )++ report_path = write_matrix_report(+ smoke_config,+ outcomes,+ target="openrouter_free_cli",+ filename_prefix="openrouter-free-cli",+ )+ failures = regression_failures(outcomes)+ assert not failures, (+ f"OpenRouter free CLI matrix regressions written to {report_path}:\n"+ + "\n".join(failures)+ )+++def _slug(value: str) -> str:+ return "".join(char if char.isalnum() else "-" for char in value).strip("-")diff --git a/tests/contracts/test_nvidia_nim_cli_matrix.py b/tests/contracts/test_nvidia_nim_cli_matrix.pyindex 70a622dc..57959b36 100644--- a/tests/contracts/test_nvidia_nim_cli_matrix.py+++ b/tests/contracts/test_nvidia_nim_cli_matrix.py@@ -4,13 +4,15 @@ import jsonfrom pathlib import Pathfrom config.settings import Settings-from smoke.lib.config import DEFAULT_TARGETS, SmokeConfig-from smoke.lib.nvidia_nim_cli import (+from smoke.lib.claude_cli_matrix import (ClaudeCliRun,+ _build_claude_cli_command,+ _subagent_probe_options,make_outcome,regression_failures,write_matrix_report,)+from smoke.lib.config import DEFAULT_TARGETS, SmokeConfigdef _smoke_config(tmp_path: Path) -> SmokeConfig:@@ -51,7 +53,12 @@ def test_nvidia_nim_cli_matrix_report_shape_and_redaction(log_path=tmp_path / "server.log",)- path = write_matrix_report(_smoke_config(tmp_path), [outcome])+ path = write_matrix_report(+ _smoke_config(tmp_path),+ [outcome],+ target="nvidia_nim_cli",+ filename_prefix="nvidia-nim-cli",+ )payload = json.loads(path.read_text(encoding="utf-8"))assert path.name.startswith("nvidia-nim-cli-matrix-test-worker-")@@ -62,9 +69,56 @@ def test_nvidia_nim_cli_matrix_report_shape_and_redaction(assert saved["classification"] == "passed"assert saved["request_count"] == 1assert saved["token_evidence"]["marker_present"] is True+ assert saved["token_evidence"]["agent_catalog_present"] is False+ assert saved["token_evidence"]["agent_tool_count"] == 0+ assert saved["token_evidence"]["agent_result_count"] == 0assert "secret-nim-key" not in path.read_text(encoding="utf-8")+def test_openrouter_free_cli_matrix_report_shape_and_redaction(+ tmp_path: Path, monkeypatch+) -> None:+ monkeypatch.setenv("OPENROUTER_API_KEY", "secret-openrouter-key")+ run = ClaudeCliRun(+ command=("claude", "-p", "redacted"),+ returncode=0,+ stdout="FCC_OPENROUTER_FREE_BASIC secret-openrouter-key",+ stderr="",+ duration_s=1.25,+ )+ outcome = make_outcome(+ model="openai/gpt-oss-120b:free",+ full_model="open_router/openai/gpt-oss-120b:free",+ source="openrouter_free_cli_default",+ feature="basic_text",+ marker="FCC_OPENROUTER_FREE_BASIC",+ run=run,+ log_delta='POST /v1/messages HTTP/1.1" 200 OK secret-openrouter-key',+ log_path=tmp_path / "server.log",+ )++ path = write_matrix_report(+ _smoke_config(tmp_path),+ [outcome],+ target="openrouter_free_cli",+ filename_prefix="openrouter-free-cli",+ )+ payload = json.loads(path.read_text(encoding="utf-8"))++ assert path.name.startswith("openrouter-free-cli-matrix-test-worker-")+ assert payload["target"] == "openrouter_free_cli"+ assert payload["models"] == ["open_router/openai/gpt-oss-120b:free"]+ saved = payload["outcomes"][0]+ assert saved["feature"] == "basic_text"+ assert saved["classification"] == "passed"+ assert saved["request_count"] == 1+ assert saved["token_evidence"]["marker_present"] is True+ assert saved["token_evidence"]["agent_catalog_present"] is False+ assert saved["token_evidence"]["agent_tool_count"] == 0+ assert saved["token_evidence"]["agent_result_count"] == 0+ assert "secret-openrouter-key" not in path.read_text(encoding="utf-8")++def test_nvidia_nim_cli_matrix_regression_detection(tmp_path: Path) -> None:run = ClaudeCliRun(command=("claude", "-p", "x"),@@ -143,6 +197,140 @@ def test_nvidia_nim_cli_raw_payload_log_counts_as_proxy_request(assert regression_failures([outcome]) == []+def test_cli_matrix_missing_agent_catalog_is_harness_bug(tmp_path: Path) -> None:+ run = ClaudeCliRun(+ command=("claude", "-p", "x"),+ returncode=0,+ stdout="ordinary answer",+ stderr="",+ duration_s=0.1,+ )+ outcome = make_outcome(+ model="openai/gpt-oss-120b:free",+ full_model="open_router/openai/gpt-oss-120b:free",+ source="openrouter_free_cli_default",+ feature="subagent_task",+ marker="FCC_OPENROUTER_FREE_TASK",+ run=run,+ log_delta=(+ "API_REQUEST: request_id=req_1 model=openai/gpt-oss-120b:free "+ "messages=1\n"+ "FULL_PAYLOAD [req_1]: {'messages': [], 'tools': [{'name': 'Read'}], "+ "'tool_choice': None}"+ ),+ log_path=tmp_path / "server.log",+ requires_agent=True,+ )++ assert outcome.classification == "harness_bug"+ assert outcome.token_evidence["agent_catalog_present"] is False+++def test_cli_matrix_agent_catalog_without_agent_use_is_model_feature_failure(+ tmp_path: Path,+) -> None:+ marker = "FCC_OPENROUTER_FREE_TASK"+ run = ClaudeCliRun(+ command=("claude", "-p", "x"),+ returncode=0,+ stdout=(+ f'{marker}\n{{"type":"tool_use","name":"Read"}}\n{{"type":"tool_result"}}'+ ),+ stderr="",+ duration_s=0.1,+ )+ outcome = make_outcome(+ model="openai/gpt-oss-120b:free",+ full_model="open_router/openai/gpt-oss-120b:free",+ source="openrouter_free_cli_default",+ feature="subagent_task",+ marker=marker,+ run=run,+ log_delta=(+ "API_REQUEST: request_id=req_1 model=openai/gpt-oss-120b:free "+ "messages=1\n"+ "FULL_PAYLOAD [req_1]: {'messages': [], 'tools': "+ "[{'name': 'Agent'}, {'name': 'Read'}], 'tool_choice': None}"+ ),+ log_path=tmp_path / "server.log",+ requires_tool_result=True,+ requires_agent=True,+ )++ assert outcome.classification == "model_feature_failure"+ assert outcome.token_evidence["agent_catalog_present"] is True+ assert outcome.token_evidence["agent_tool_count"] == 0+++def test_cli_matrix_agent_use_result_and_marker_pass(tmp_path: Path) -> None:+ marker = "FCC_OPENROUTER_FREE_TASK"+ run = ClaudeCliRun(+ command=("claude", "-p", "x"),+ returncode=0,+ stdout=(+ f'{marker}\n{{"type":"tool_use","name":"Agent"}}\n'+ '{"type":"tool_result","content":"agentId: abc123"}'+ ),+ stderr="",+ duration_s=0.1,+ )+ outcome = make_outcome(+ model="openai/gpt-oss-120b:free",+ full_model="open_router/openai/gpt-oss-120b:free",+ source="openrouter_free_cli_default",+ feature="subagent_task",+ marker=marker,+ run=run,+ log_delta=(+ "API_REQUEST: request_id=req_1 model=openai/gpt-oss-120b:free "+ "messages=1\n"+ "FULL_PAYLOAD [req_1]: {'messages': [], 'tools': "+ "[{'name': 'Agent'}, {'name': 'Read'}], 'tool_choice': None}"+ ),+ log_path=tmp_path / "server.log",+ requires_tool_result=True,+ requires_agent=True,+ )++ assert outcome.classification == "passed"+ assert outcome.token_evidence["agent_catalog_present"] is True+ assert outcome.token_evidence["agent_tool_count"] == 1+ assert outcome.token_evidence["agent_result_count"] == 1+++def test_cli_matrix_agent_prompt_text_without_tool_evidence_does_not_pass(+ tmp_path: Path,+) -> None:+ marker = "FCC_OPENROUTER_FREE_TASK"+ run = ClaudeCliRun(+ command=("claude", "-p", "x"),+ returncode=0,+ stdout=f"{marker}\nAgent should read the file.",+ stderr="",+ duration_s=0.1,+ )+ outcome = make_outcome(+ model="openai/gpt-oss-120b:free",+ full_model="open_router/openai/gpt-oss-120b:free",+ source="openrouter_free_cli_default",+ feature="subagent_task",+ marker=marker,+ run=run,+ log_delta=(+ "API_REQUEST: request_id=req_1 model=openai/gpt-oss-120b:free "+ "messages=1\n"+ "FULL_PAYLOAD [req_1]: {'messages': [], 'tools': "+ "[{'name': 'Agent'}, {'name': 'Read'}], 'tool_choice': None}"+ ),+ log_path=tmp_path / "server.log",+ requires_agent=True,+ )++ assert outcome.classification == "model_feature_failure"+ assert outcome.token_evidence["agent_catalog_present"] is True+ assert outcome.token_evidence["agent_tool_count"] == 0++def test_nvidia_nim_cli_timeout_is_not_model_missing(tmp_path: Path,) -> None:@@ -194,3 +382,89 @@ def test_nvidia_nim_cli_success_beats_verbose_timeout_words(tmp_path: Path) -> Nassert outcome.classification == "passed"assert outcome.request_count == 1+++def test_cli_matrix_uuid_429_does_not_count_as_upstream_unavailable(+ tmp_path: Path,+) -> None:+ run = ClaudeCliRun(+ command=("claude", "-p", "x"),+ returncode=0,+ stdout='{"uuid":"d3c76eea-3634-4299-aec0-e7634b3716da"}',+ stderr="",+ duration_s=0.1,+ )+ outcome = make_outcome(+ model="openai/gpt-oss-120b:free",+ full_model="open_router/openai/gpt-oss-120b:free",+ source="openrouter_free_cli_default",+ feature="subagent_task",+ marker="FCC_OPENROUTER_FREE_TASK",+ run=run,+ log_delta="API_REQUEST: request_id=req_1 model=openai/gpt-oss-120b:free messages=2",+ log_path=tmp_path / "server.log",+ requires_task=True,+ )++ assert outcome.classification == "model_feature_failure"+++def test_cli_matrix_real_http_429_counts_as_upstream_unavailable(+ tmp_path: Path,+) -> None:+ run = ClaudeCliRun(+ command=("claude", "-p", "x"),+ returncode=0,+ stdout="ordinary answer",+ stderr="",+ duration_s=0.1,+ )+ outcome = make_outcome(+ model="openai/gpt-oss-120b:free",+ full_model="open_router/openai/gpt-oss-120b:free",+ source="openrouter_free_cli_default",+ feature="subagent_task",+ marker="FCC_OPENROUTER_FREE_TASK",+ run=run,+ log_delta=(+ "API_REQUEST: request_id=req_1 model=openai/gpt-oss-120b:free "+ 'messages=2 upstream HTTP/1.1" 429 Too Many Requests'+ ),+ log_path=tmp_path / "server.log",+ requires_task=True,+ )++ assert outcome.classification == "upstream_unavailable"+++def test_cli_matrix_default_command_uses_bare_mode() -> None:+ command = _build_claude_cli_command(+ claude_bin="claude",+ prompt="hello",+ tools="Read",+ )++ assert command[:2] == ("claude", "--bare")+ assert "--tools" in command+ assert "Read" in command+++def test_cli_matrix_subagent_command_uses_agent_without_bare_or_task() -> None:+ bare, tools, pre_tool_args, extra_args = _subagent_probe_options("{}")+ command = _build_claude_cli_command(+ claude_bin="claude",+ prompt="hello",+ tools=tools,+ bare=bare,+ pre_tool_args=pre_tool_args,+ extra_args=extra_args,+ )++ assert "--bare" not in command+ assert command[command.index("--setting-sources") + 1] == "local"+ assert "--strict-mcp-config" in command+ assert command[command.index("--mcp-config") + 1] == '{"mcpServers":{}}'+ assert command[command.index("--tools") + 1] == "Agent,Read"+ assert command[command.index("--allowedTools") + 1] == "Agent,Read"+ assert command[command.index("--agents") + 1] == "{}"+ assert "Task,Read" not in commanddiff --git a/tests/contracts/test_smoke_config.py b/tests/contracts/test_smoke_config.pyindex a687ded3..3b368aa3 100644--- a/tests/contracts/test_smoke_config.py+++ b/tests/contracts/test_smoke_config.py@@ -7,11 +7,13 @@ from smoke.lib.config import (ALL_TARGETS,DEFAULT_TARGETS,NVIDIA_NIM_CLI_DEFAULT_MODELS,+ OPENROUTER_FREE_CLI_DEFAULT_MODELS,OPT_IN_TARGETS,PROVIDER_SMOKE_DEFAULT_MODELS,TARGET_REQUIRED_ENV,SmokeConfig,nvidia_nim_cli_model_refs,+ openrouter_free_cli_model_refs,)@@ -61,6 +63,10 @@ def test_nvidia_nim_cli_is_opt_in_smoke_target() -> None:assert "nvidia_nim_cli" in OPT_IN_TARGETSassert "nvidia_nim_cli" in ALL_TARGETSassert "nvidia_nim_cli" in TARGET_REQUIRED_ENV+ assert "openrouter_free_cli" not in DEFAULT_TARGETS+ assert "openrouter_free_cli" in OPT_IN_TARGETS+ assert "openrouter_free_cli" in ALL_TARGETS+ assert "openrouter_free_cli" in TARGET_REQUIRED_ENVdef test_ollama_provider_configuration_uses_base_url() -> None:@@ -265,3 +271,77 @@ def test_smoke_config_returns_nvidia_nim_cli_provider_models(monkeypatch) -> Nonassert models[0].provider == "nvidia_nim"assert models[0].full_model == "nvidia_nim/z-ai/glm-5.1"assert models[0].source == "nvidia_nim_cli_default"+++def test_openrouter_free_cli_default_models_are_normalized() -> None:+ refs = openrouter_free_cli_model_refs({})++ assert tuple(refs) == tuple(+ f"open_router/{model}" for model in OPENROUTER_FREE_CLI_DEFAULT_MODELS+ )+ assert "open_router/nvidia/nemotron-3-super-120b-a12b:free" in refs+ assert "open_router/poolside/laguna-m.1:free" in refs+ assert set(refs.values()) == {"openrouter_free_cli_default"}+++def test_openrouter_free_cli_models_override_and_append() -> None:+ refs = openrouter_free_cli_model_refs(+ {+ "FCC_SMOKE_OPENROUTER_FREE_MODELS": (+ "openai/gpt-oss-120b:free,open_router/custom/model:free"+ ),+ "FCC_SMOKE_OPENROUTER_FREE_EXTRA_MODELS": (+ "poolside/laguna-m.1:free,openai/gpt-oss-120b:free"+ ),+ }+ )++ assert tuple(refs) == (+ "open_router/openai/gpt-oss-120b:free",+ "open_router/custom/model:free",+ "open_router/poolside/laguna-m.1:free",+ )+ assert refs["open_router/openai/gpt-oss-120b:free"] == (+ "FCC_SMOKE_OPENROUTER_FREE_MODELS"+ )+ assert refs["open_router/poolside/laguna-m.1:free"] == (+ "FCC_SMOKE_OPENROUTER_FREE_EXTRA_MODELS"+ )+++def test_openrouter_free_cli_models_reject_empty_override() -> None:+ try:+ openrouter_free_cli_model_refs({"FCC_SMOKE_OPENROUTER_FREE_MODELS": " , "})+ except ValueError as exc:+ assert "FCC_SMOKE_OPENROUTER_FREE_MODELS" in str(exc)+ else:+ raise AssertionError("expected empty OpenRouter free CLI override to fail")+++def test_openrouter_free_cli_models_reject_wrong_provider_prefix() -> None:+ try:+ openrouter_free_cli_model_refs(+ {"FCC_SMOKE_OPENROUTER_FREE_MODELS": "nvidia_nim/model"}+ )+ except ValueError as exc:+ assert "open_router" in str(exc)+ else:+ raise AssertionError("expected wrong provider prefix to fail")+++def test_smoke_config_returns_openrouter_free_cli_provider_models(monkeypatch) -> None:+ monkeypatch.delenv("FCC_SMOKE_OPENROUTER_FREE_MODELS", raising=False)+ monkeypatch.delenv("FCC_SMOKE_OPENROUTER_FREE_EXTRA_MODELS", raising=False)+ config = _smoke_config(+ settings=_settings(+ model="open_router/openai/gpt-oss-120b:free",+ open_router_api_key="openrouter-key",+ ollama_base_url="",+ )+ )++ models = config.openrouter_free_cli_models()++ assert models[0].provider == "open_router"+ assert models[0].full_model == "open_router/nvidia/nemotron-3-super-120b-a12b:free"+ assert models[0].source == "openrouter_free_cli_default"