Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.
Rust
feat(launch): startup-timing breakdown and faster AUTO backend-format detection (#70)
Elyas Mehtabuddin9dcccfeb2fde
diff --git a/docs/architecture.md b/docs/architecture.mdindex 2b2eed4f5..ccf0a46db 100644--- a/docs/architecture.md+++ b/docs/architecture.md@@ -58,17 +58,22 @@ OpenClaw is intentionally pinned to `OPENAI` for its equivalent target.```mermaidflowchart TBauto["BackendFormat.AUTO"]+ chat{"/v1/chat/completions works?"}+ openai_chat["OPENAI<br/>/v1/chat/completions"]messages{"/v1/messages works?"}anthropic["ANTHROPIC<br/>/v1/messages"]responses{"/v1/responses works?"}responses_format["RESPONSES<br/>/v1/responses"]- openai["OPENAI<br/>/v1/chat/completions fallback"]+ openai_fallback["OPENAI<br/>/v1/chat/completions fallback"]- auto -->|"Probe /v1/messages"| messages+ auto -->|"Probe /v1/chat/completions"| chat+ chat -->|"Timeout: assume Chat Completions"| openai_chat+ chat -->|"Yes"| openai_chat+ chat -->|"No (fast 404): probe /v1/messages"| messagesmessages -->|"Yes"| anthropicmessages -->|"No: probe /v1/responses"| responsesresponses -->|"Yes"| responses_format- responses -->|"No"| openai+ responses -->|"No"| openai_fallback```Supported inbound and response formats are handled automatically.@@ -82,6 +87,14 @@ upstream format.> Prefer an explicit format for controlled deployments. It skips capability> probes and makes the upstream contract clear. Use `AUTO` when provider> capabilities are unknown or vary across deployments.+>+> **`AUTO` costs startup latency.** Each probe is a live request to the upstream+> made before the agent starts, so a slow endpoint adds a round-trip per probe+> (up to three, tried in order). This is the main reason a launch that uses+> `AUTO` (Claude Code, Codex) starts slower than one pinned to an explicit+> format. Set `format:` explicitly to remove the probing entirely. To see the+> per-probe cost of a launch, run it with `--startup-timing` (or+> `SWITCHYARD_STARTUP_TIMING=1`), which prints each probe on its own line.## Related Documentation- [Getting Started](getting_started.md): install Switchyard and run a first requestdiff --git a/switchyard/cli/launch_command.py b/switchyard/cli/launch_command.pyindex abb156655..30d7e1b0d 100644--- a/switchyard/cli/launch_command.py+++ b/switchyard/cli/launch_command.py@@ -35,6 +35,7 @@ from switchyard.cli.routing import (build_deterministic_routing_config,require_route_model,)+from switchyard.lib import startup_timingfrom switchyard.lib.backends.llm_target import BackendFormatfrom switchyard.lib.profiles.random_routing import (RandomRoutingConfig,@@ -480,6 +481,9 @@ def cmd_launch_claude(args: argparse.Namespace) -> None:Random / latency-aware routing live in the YAML schema."""+ if getattr(args, "startup_timing", False):+ startup_timing.enable()+ startup_timing.mark("launch invoked")if args.routing_profiles and getattr(args, "smoke", False):raise SystemExit("launch claude: --smoke and --routing-profiles cannot be combined. "@@ -626,6 +630,7 @@ def cmd_launch_claude(args: argparse.Namespace) -> None:intake = resolve_launch_intake_config(args, target="claude", default_app="claude-code-switchyard",)+ startup_timing.mark("config + credentials resolved")if deterministic:primary_connectivity = require_launch_tier_key(diff --git a/switchyard/cli/launchers/claude_code_launcher.py b/switchyard/cli/launchers/claude_code_launcher.pyindex a7c284097..148764c13 100644--- a/switchyard/cli/launchers/claude_code_launcher.py+++ b/switchyard/cli/launchers/claude_code_launcher.py@@ -69,6 +69,7 @@ from switchyard.cli.launchers.session_summary import print_session_summaryfrom switchyard.cli.route_bundle import (load_route_bundle_table,)+from switchyard.lib import startup_timingfrom switchyard.lib.backends.llm_target import (BackendFormat,LlmTarget,@@ -316,6 +317,7 @@ def _run_claude_with_switchyard(Returns the ``claude`` process's exit code (or ``127`` if thebinary wasn't found, ``130`` on Ctrl-C)."""+ startup_timing.mark("chain assembled")# Expose every table entry under a `claude-` prefixed alias so Claude# Code's gateway-discovery filter accepts the full listing. Originals stay# registered for direct-id callers; aliases share the same chain object.@@ -339,6 +341,7 @@ def _run_claude_with_switchyard(thread.is_alive(),server.started,)+ startup_timing.mark("proxy thread started")try:if not _wait_ready(resolved_port):@@ -355,6 +358,7 @@ def _run_claude_with_switchyard()return 1+ startup_timing.mark("proxy health-ready")suppress_uvicorn_stream_handlers()logger.info("proxy ready on port %d", resolved_port)if intake is not None:@@ -386,6 +390,8 @@ def _run_claude_with_switchyard(env_overrides.get("ANTHROPIC_CUSTOM_MODEL_OPTION"),env_overrides.get("CLAUDE_CODE_ENABLE_GATEWAY_MODEL_DISCOVERY"),)+ startup_timing.mark("child agent spawned")+ startup_timing.dump()if stdin_is_tty():footer = LiveStatsFooter(stats, display_model, health, table=table,diff --git a/switchyard/cli/switchyard_cli.py b/switchyard/cli/switchyard_cli.pyindex 660f5c62c..3321fe5e5 100644--- a/switchyard/cli/switchyard_cli.py+++ b/switchyard/cli/switchyard_cli.py@@ -1168,6 +1168,14 @@ def _build_parser() -> argparse.ArgumentParser:"--timeout", type=float, default=None,help="Request timeout in seconds for the backend LLM client",)+ lc.add_argument(+ "--startup-timing", action="store_true",+ help=(+ "Print a per-stage startup timing breakdown to stderr before "+ "claude starts (each backend-format probe on its own line). "+ "Also enabled with SWITCHYARD_STARTUP_TIMING=1."+ ),+ )_add_launch_deterministic_override_args(lc)_add_launch_intake_args(lc)lc.add_argument(diff --git a/switchyard/lib/backends/backend_format_resolver.py b/switchyard/lib/backends/backend_format_resolver.pyindex c85604a65..e74ffa10c 100644--- a/switchyard/lib/backends/backend_format_resolver.py+++ b/switchyard/lib/backends/backend_format_resolver.py@@ -21,8 +21,6 @@ a neutral IR, so all (inbound, backend) combinations are valid regardless ofwhich format the client uses."""-from __future__ import annotations-import jsonimport loggingfrom dataclasses import dataclass@@ -30,6 +28,7 @@ from typing import Anyimport httpx+from switchyard.lib import startup_timingfrom switchyard.lib.backends.llm_target import (BackendFormat,LlmTarget,@@ -85,34 +84,60 @@ class BackendFormatResolver:timeout_s = tier.endpoint.timeout_secs or _DEFAULT_TIMEOUT_S- if probe_openai_chat_completions_support_sync(- base_url=tier.endpoint.base_url,- api_key=tier.endpoint.api_key,- model=tier.model,- timeout_s=timeout_s,- ):+ # startup_timing marks let `switchyard launch --startup-timing` show each+ # probe as its own line, so a slow AUTO detection names which route stalled.+ startup_timing.mark("chain init")++ # Chat Completions is probed first: it is the most widely supported+ # format and the universal fallback. A transport timeout (the endpoint+ # is reachable but slow, e.g. a cold NVCF/Azure deployment) is a+ # different signal from a fast "not wired" 404. On a timeout, assume+ # Chat Completions rather than serially probing the rarer /v1/messages+ # and /v1/responses routes — each of those would stack another timeout+ # and turn one slow startup into several seconds. A 404 (even a slow+ # one) still falls through to the probes below, so Anthropic-native and+ # Responses-only endpoints are detected as before.+ try:+ chat_supported = probe_openai_chat_completions_support_sync(+ base_url=tier.endpoint.base_url,+ api_key=tier.endpoint.api_key,+ model=tier.model,+ timeout_s=timeout_s,+ )+ except httpx.TimeoutException:+ startup_timing.mark("probe: /v1/chat/completions (timed out)")+ return BackendFormatResolution(+ format=BackendFormat.OPENAI,+ reason="chat-completions probe timed out; assuming Chat Completions",+ )+ startup_timing.mark("probe: /v1/chat/completions")+ if chat_supported:return BackendFormatResolution(format=BackendFormat.OPENAI,reason="upstream /v1/chat/completions probe succeeded",)- if probe_anthropic_messages_support_sync(+ anthropic_supported = probe_anthropic_messages_support_sync(base_url=tier.endpoint.base_url,api_key=tier.endpoint.api_key,model=tier.model,timeout_s=timeout_s,- ):+ )+ startup_timing.mark("probe: /v1/messages")+ if anthropic_supported:return BackendFormatResolution(format=BackendFormat.ANTHROPIC,reason="upstream /v1/messages probe succeeded; Chat Completions not available",)- if probe_openai_responses_support_sync(+ responses_supported = probe_openai_responses_support_sync(base_url=tier.endpoint.base_url,api_key=tier.endpoint.api_key,model=tier.model,timeout_s=timeout_s,- ):+ )+ startup_timing.mark("probe: /v1/responses")+ if responses_supported:return BackendFormatResolution(format=BackendFormat.RESPONSES,reason="upstream /v1/responses probe succeeded; Chat Completions not available",@@ -153,6 +178,12 @@ def probe_openai_chat_completions_support_sync(Sends a minimal-body probe POST with Bearer auth. A 404 means theChat Completions endpoint is not wired — the caller should probeAnthropic Messages or Responses next.++ Raises ``httpx.TimeoutException`` on a transport timeout. A timeout means+ the endpoint is reachable but slow (e.g. a cold NVCF/Azure deployment),+ which is a different signal from a fast "not wired" 404: the resolver+ stops probing and assumes Chat Completions instead of stacking the slower+ ``/v1/messages`` and ``/v1/responses`` probes."""url = f"{base_url.rstrip('/')}/chat/completions"headers = {"Authorization": f"Bearer {api_key}", "content-type": "application/json"}@@ -164,6 +195,8 @@ def probe_openai_chat_completions_support_sync(try:with httpx.Client(timeout=timeout_s) as client:resp = client.post(url, headers=headers, json=req_body)+ except httpx.TimeoutException:+ raiseexcept httpx.RequestError as e:log.warning("OpenAI /v1/chat/completions probe failed (%s); "diff --git a/switchyard/lib/startup_timing.py b/switchyard/lib/startup_timing.pynew file mode 100644index 000000000..bff550e42--- /dev/null+++ b/switchyard/lib/startup_timing.py@@ -0,0 +1,56 @@+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.+# SPDX-License-Identifier: Apache-2.0++"""Per-stage startup timing for ``switchyard launch``.++Turn it on with ``--startup-timing`` or ``SWITCHYARD_STARTUP_TIMING=1``; it does+nothing when off. ``mark()`` records a checkpoint during startup, and ``dump()``+prints the time between checkpoints to stderr just before the agent starts. It+times only switchyard's own startup work, not Python's one-time import cost.+"""++import os+import sys+import time+from typing import TextIO++# On unless SWITCHYARD_STARTUP_TIMING is unset or "0", or once --startup-timing calls+# enable(). Read once at import — the env var does not change during a launch.+enabled: bool = os.environ.get("SWITCHYARD_STARTUP_TIMING", "0") != "0"++# (label, perf_counter timestamp) for each point reached during startup.+_marks: list[tuple[str, float]] = []+++def enable() -> None:+ """Turn timing on for this process (called by the ``--startup-timing`` flag)."""+ global enabled+ enabled = True+++def mark(label: str) -> None:+ """Record that startup reached *label*. No-op unless timing is enabled."""+ if enabled:+ _marks.append((label, time.perf_counter()))+++def dump(stream: TextIO | None = None) -> None:+ """Print the per-stage breakdown to stderr, then reset. No-op when disabled.++ Each line is the time between consecutive marks; the last line is the total+ from the first mark to the last.+ """+ if not enabled or len(_marks) < 2:+ _marks.clear()+ return+ out = stream if stream is not None else sys.stderr+ start = _marks[0][1]+ prev = start+ lines = ["switchyard startup timing:"]+ for label, stamp in _marks[1:]:+ lines.append(f" {(stamp - prev) * 1000:8.1f} ms {label}")+ prev = stamp+ lines.append(f" {'-' * 8}")+ lines.append(f" {(prev - start) * 1000:8.1f} ms total (launch invoked -> child spawn)")+ print("\n".join(lines), file=out)+ _marks.clear()diff --git a/tests/test_backend_format_resolver.py b/tests/test_backend_format_resolver.pyindex 9ce611434..4d1b1f714 100644--- a/tests/test_backend_format_resolver.py+++ b/tests/test_backend_format_resolver.py@@ -1,10 +1,10 @@# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.# SPDX-License-Identifier: Apache-2.0-from __future__ import annotations-+import httpximport pytest+from switchyard.lib import startup_timingfrom switchyard.lib.backends import (backend_format_resolver as resolver_mod,)@@ -217,6 +217,67 @@ def test_auto_falls_back_to_openai_when_all_probes_fail(assert resolution.format is BackendFormat.OPENAI+def test_auto_chat_completions_timeout_assumes_openai(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ """A Chat Completions probe that times out (raises, not a fast 404) resolves+ to OPENAI without stacking the slower /v1/messages and /v1/responses probes.+ A 404 returns False and still falls through — that path is covered by+ test_auto_resolves_to_anthropic_when_chat_completions_unavailable."""+ def timed_out(**_: object) -> bool:+ raise httpx.ReadTimeout("probe timed out")++ monkeypatch.setattr(resolver_mod, "probe_openai_chat_completions_support_sync",+ timed_out)+ monkeypatch.setattr(resolver_mod, "probe_anthropic_messages_support_sync",+ _no_probe("anthropic"))+ monkeypatch.setattr(resolver_mod, "probe_openai_responses_support_sync",+ _no_probe("responses"))++ resolution = resolver_mod.BackendFormatResolver.resolve(+ LlmTarget(+ model="slow-endpoint-model",+ format=BackendFormat.AUTO,+ base_url="https://slow.test/v1",+ api_key="sk-test", # pragma: allowlist secret+ timeout_secs=0.05,+ ),+ )++ assert resolution.format is BackendFormat.OPENAI+ assert "timed out" in resolution.reason+++def test_auto_records_each_probe_in_startup_timing(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ """With timing on, each probe reached leaves its own mark so+ `launch --startup-timing` can show a per-route breakdown."""+ monkeypatch.setattr(startup_timing, "enabled", True)+ startup_timing._marks.clear()++ monkeypatch.setattr(resolver_mod, "probe_openai_chat_completions_support_sync",+ _RecordingProbe(result=False))+ monkeypatch.setattr(resolver_mod, "probe_anthropic_messages_support_sync",+ _RecordingProbe(result=True))+ monkeypatch.setattr(resolver_mod, "probe_openai_responses_support_sync",+ _no_probe("responses"))++ resolution = resolver_mod.BackendFormatResolver.resolve(+ LlmTarget(+ model="some-model",+ format=BackendFormat.AUTO,+ base_url="https://api.anthropic.com/v1",+ api_key="sk-test", # pragma: allowlist secret+ ),+ )++ assert resolution.format is BackendFormat.ANTHROPIC+ labels = [label for label, _ in startup_timing._marks]+ assert labels == ["chain init", "probe: /v1/chat/completions", "probe: /v1/messages"]+ startup_timing._marks.clear()++# ---------------------------------------------------------------------------# Missing inputs# ---------------------------------------------------------------------------