Skip to content

Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.

Rust

feat(launch): startup-timing breakdown and faster AUTO backend-format detection (#70)

Elyas Mehtabuddin9dcccfeb2fde
diff --git a/docs/architecture.md b/docs/architecture.md
index 2b2eed4f5..ccf0a46db 100644
--- a/docs/architecture.md
+++ b/docs/architecture.md
@@ -58,17 +58,22 @@ OpenClaw is intentionally pinned to `OPENAI` for its equivalent target.
```mermaid
flowchart TB
auto["BackendFormat.AUTO"]
+ chat{"/v1/chat/completions works?"}
+ openai_chat["OPENAI<br/>/v1/chat/completions"]
messages{"/v1/messages works?"}
anthropic["ANTHROPIC<br/>/v1/messages"]
responses{"/v1/responses works?"}
responses_format["RESPONSES<br/>/v1/responses"]
- openai["OPENAI<br/>/v1/chat/completions fallback"]
+ openai_fallback["OPENAI<br/>/v1/chat/completions fallback"]
- auto -->|"Probe /v1/messages"| messages
+ auto -->|"Probe /v1/chat/completions"| chat
+ chat -->|"Timeout: assume Chat Completions"| openai_chat
+ chat -->|"Yes"| openai_chat
+ chat -->|"No (fast 404): probe /v1/messages"| messages
messages -->|"Yes"| anthropic
messages -->|"No: probe /v1/responses"| responses
responses -->|"Yes"| responses_format
- responses -->|"No"| openai
+ responses -->|"No"| openai_fallback
```
Supported inbound and response formats are handled automatically.
@@ -82,6 +87,14 @@ upstream format.
> Prefer an explicit format for controlled deployments. It skips capability
> probes and makes the upstream contract clear. Use `AUTO` when provider
> capabilities are unknown or vary across deployments.
+>
+> **`AUTO` costs startup latency.** Each probe is a live request to the upstream
+> made before the agent starts, so a slow endpoint adds a round-trip per probe
+> (up to three, tried in order). This is the main reason a launch that uses
+> `AUTO` (Claude Code, Codex) starts slower than one pinned to an explicit
+> format. Set `format:` explicitly to remove the probing entirely. To see the
+> per-probe cost of a launch, run it with `--startup-timing` (or
+> `SWITCHYARD_STARTUP_TIMING=1`), which prints each probe on its own line.
## Related Documentation
- [Getting Started](getting_started.md): install Switchyard and run a first request
diff --git a/switchyard/cli/launch_command.py b/switchyard/cli/launch_command.py
index abb156655..30d7e1b0d 100644
--- a/switchyard/cli/launch_command.py
+++ b/switchyard/cli/launch_command.py
@@ -35,6 +35,7 @@ from switchyard.cli.routing import (
build_deterministic_routing_config,
require_route_model,
)
+from switchyard.lib import startup_timing
from switchyard.lib.backends.llm_target import BackendFormat
from switchyard.lib.profiles.random_routing import (
RandomRoutingConfig,
@@ -480,6 +481,9 @@ def cmd_launch_claude(args: argparse.Namespace) -> None:
Random / latency-aware routing live in the YAML schema.
"""
+ if getattr(args, "startup_timing", False):
+ startup_timing.enable()
+ startup_timing.mark("launch invoked")
if args.routing_profiles and getattr(args, "smoke", False):
raise SystemExit(
"launch claude: --smoke and --routing-profiles cannot be combined. "
@@ -626,6 +630,7 @@ def cmd_launch_claude(args: argparse.Namespace) -> None:
intake = resolve_launch_intake_config(
args, target="claude", default_app="claude-code-switchyard",
)
+ startup_timing.mark("config + credentials resolved")
if deterministic:
primary_connectivity = require_launch_tier_key(
diff --git a/switchyard/cli/launchers/claude_code_launcher.py b/switchyard/cli/launchers/claude_code_launcher.py
index a7c284097..148764c13 100644
--- a/switchyard/cli/launchers/claude_code_launcher.py
+++ b/switchyard/cli/launchers/claude_code_launcher.py
@@ -69,6 +69,7 @@ from switchyard.cli.launchers.session_summary import print_session_summary
from switchyard.cli.route_bundle import (
load_route_bundle_table,
)
+from switchyard.lib import startup_timing
from switchyard.lib.backends.llm_target import (
BackendFormat,
LlmTarget,
@@ -316,6 +317,7 @@ def _run_claude_with_switchyard(
Returns the ``claude`` process's exit code (or ``127`` if the
binary wasn't found, ``130`` on Ctrl-C).
"""
+ startup_timing.mark("chain assembled")
# Expose every table entry under a `claude-` prefixed alias so Claude
# Code's gateway-discovery filter accepts the full listing. Originals stay
# registered for direct-id callers; aliases share the same chain object.
@@ -339,6 +341,7 @@ def _run_claude_with_switchyard(
thread.is_alive(),
server.started,
)
+ startup_timing.mark("proxy thread started")
try:
if not _wait_ready(resolved_port):
@@ -355,6 +358,7 @@ def _run_claude_with_switchyard(
)
return 1
+ startup_timing.mark("proxy health-ready")
suppress_uvicorn_stream_handlers()
logger.info("proxy ready on port %d", resolved_port)
if intake is not None:
@@ -386,6 +390,8 @@ def _run_claude_with_switchyard(
env_overrides.get("ANTHROPIC_CUSTOM_MODEL_OPTION"),
env_overrides.get("CLAUDE_CODE_ENABLE_GATEWAY_MODEL_DISCOVERY"),
)
+ startup_timing.mark("child agent spawned")
+ startup_timing.dump()
if stdin_is_tty():
footer = LiveStatsFooter(
stats, display_model, health, table=table,
diff --git a/switchyard/cli/switchyard_cli.py b/switchyard/cli/switchyard_cli.py
index 660f5c62c..3321fe5e5 100644
--- a/switchyard/cli/switchyard_cli.py
+++ b/switchyard/cli/switchyard_cli.py
@@ -1168,6 +1168,14 @@ def _build_parser() -> argparse.ArgumentParser:
"--timeout", type=float, default=None,
help="Request timeout in seconds for the backend LLM client",
)
+ lc.add_argument(
+ "--startup-timing", action="store_true",
+ help=(
+ "Print a per-stage startup timing breakdown to stderr before "
+ "claude starts (each backend-format probe on its own line). "
+ "Also enabled with SWITCHYARD_STARTUP_TIMING=1."
+ ),
+ )
_add_launch_deterministic_override_args(lc)
_add_launch_intake_args(lc)
lc.add_argument(
diff --git a/switchyard/lib/backends/backend_format_resolver.py b/switchyard/lib/backends/backend_format_resolver.py
index c85604a65..e74ffa10c 100644
--- a/switchyard/lib/backends/backend_format_resolver.py
+++ b/switchyard/lib/backends/backend_format_resolver.py
@@ -21,8 +21,6 @@ a neutral IR, so all (inbound, backend) combinations are valid regardless of
which format the client uses.
"""
-from __future__ import annotations
-
import json
import logging
from dataclasses import dataclass
@@ -30,6 +28,7 @@ from typing import Any
import httpx
+from switchyard.lib import startup_timing
from switchyard.lib.backends.llm_target import (
BackendFormat,
LlmTarget,
@@ -85,34 +84,60 @@ class BackendFormatResolver:
timeout_s = tier.endpoint.timeout_secs or _DEFAULT_TIMEOUT_S
- if probe_openai_chat_completions_support_sync(
- base_url=tier.endpoint.base_url,
- api_key=tier.endpoint.api_key,
- model=tier.model,
- timeout_s=timeout_s,
- ):
+ # startup_timing marks let `switchyard launch --startup-timing` show each
+ # probe as its own line, so a slow AUTO detection names which route stalled.
+ startup_timing.mark("chain init")
+
+ # Chat Completions is probed first: it is the most widely supported
+ # format and the universal fallback. A transport timeout (the endpoint
+ # is reachable but slow, e.g. a cold NVCF/Azure deployment) is a
+ # different signal from a fast "not wired" 404. On a timeout, assume
+ # Chat Completions rather than serially probing the rarer /v1/messages
+ # and /v1/responses routes — each of those would stack another timeout
+ # and turn one slow startup into several seconds. A 404 (even a slow
+ # one) still falls through to the probes below, so Anthropic-native and
+ # Responses-only endpoints are detected as before.
+ try:
+ chat_supported = probe_openai_chat_completions_support_sync(
+ base_url=tier.endpoint.base_url,
+ api_key=tier.endpoint.api_key,
+ model=tier.model,
+ timeout_s=timeout_s,
+ )
+ except httpx.TimeoutException:
+ startup_timing.mark("probe: /v1/chat/completions (timed out)")
+ return BackendFormatResolution(
+ format=BackendFormat.OPENAI,
+ reason="chat-completions probe timed out; assuming Chat Completions",
+ )
+ startup_timing.mark("probe: /v1/chat/completions")
+ if chat_supported:
return BackendFormatResolution(
format=BackendFormat.OPENAI,
reason="upstream /v1/chat/completions probe succeeded",
)
- if probe_anthropic_messages_support_sync(
+ anthropic_supported = probe_anthropic_messages_support_sync(
base_url=tier.endpoint.base_url,
api_key=tier.endpoint.api_key,
model=tier.model,
timeout_s=timeout_s,
- ):
+ )
+ startup_timing.mark("probe: /v1/messages")
+ if anthropic_supported:
return BackendFormatResolution(
format=BackendFormat.ANTHROPIC,
reason="upstream /v1/messages probe succeeded; Chat Completions not available",
)
- if probe_openai_responses_support_sync(
+ responses_supported = probe_openai_responses_support_sync(
base_url=tier.endpoint.base_url,
api_key=tier.endpoint.api_key,
model=tier.model,
timeout_s=timeout_s,
- ):
+ )
+ startup_timing.mark("probe: /v1/responses")
+ if responses_supported:
return BackendFormatResolution(
format=BackendFormat.RESPONSES,
reason="upstream /v1/responses probe succeeded; Chat Completions not available",
@@ -153,6 +178,12 @@ def probe_openai_chat_completions_support_sync(
Sends a minimal-body probe POST with Bearer auth. A 404 means the
Chat Completions endpoint is not wired — the caller should probe
Anthropic Messages or Responses next.
+
+ Raises ``httpx.TimeoutException`` on a transport timeout. A timeout means
+ the endpoint is reachable but slow (e.g. a cold NVCF/Azure deployment),
+ which is a different signal from a fast "not wired" 404: the resolver
+ stops probing and assumes Chat Completions instead of stacking the slower
+ ``/v1/messages`` and ``/v1/responses`` probes.
"""
url = f"{base_url.rstrip('/')}/chat/completions"
headers = {"Authorization": f"Bearer {api_key}", "content-type": "application/json"}
@@ -164,6 +195,8 @@ def probe_openai_chat_completions_support_sync(
try:
with httpx.Client(timeout=timeout_s) as client:
resp = client.post(url, headers=headers, json=req_body)
+ except httpx.TimeoutException:
+ raise
except httpx.RequestError as e:
log.warning(
"OpenAI /v1/chat/completions probe failed (%s); "
diff --git a/switchyard/lib/startup_timing.py b/switchyard/lib/startup_timing.py
new file mode 100644
index 000000000..bff550e42
--- /dev/null
+++ b/switchyard/lib/startup_timing.py
@@ -0,0 +1,56 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+"""Per-stage startup timing for ``switchyard launch``.
+
+Turn it on with ``--startup-timing`` or ``SWITCHYARD_STARTUP_TIMING=1``; it does
+nothing when off. ``mark()`` records a checkpoint during startup, and ``dump()``
+prints the time between checkpoints to stderr just before the agent starts. It
+times only switchyard's own startup work, not Python's one-time import cost.
+"""
+
+import os
+import sys
+import time
+from typing import TextIO
+
+# On unless SWITCHYARD_STARTUP_TIMING is unset or "0", or once --startup-timing calls
+# enable(). Read once at import — the env var does not change during a launch.
+enabled: bool = os.environ.get("SWITCHYARD_STARTUP_TIMING", "0") != "0"
+
+# (label, perf_counter timestamp) for each point reached during startup.
+_marks: list[tuple[str, float]] = []
+
+
+def enable() -> None:
+ """Turn timing on for this process (called by the ``--startup-timing`` flag)."""
+ global enabled
+ enabled = True
+
+
+def mark(label: str) -> None:
+ """Record that startup reached *label*. No-op unless timing is enabled."""
+ if enabled:
+ _marks.append((label, time.perf_counter()))
+
+
+def dump(stream: TextIO | None = None) -> None:
+ """Print the per-stage breakdown to stderr, then reset. No-op when disabled.
+
+ Each line is the time between consecutive marks; the last line is the total
+ from the first mark to the last.
+ """
+ if not enabled or len(_marks) < 2:
+ _marks.clear()
+ return
+ out = stream if stream is not None else sys.stderr
+ start = _marks[0][1]
+ prev = start
+ lines = ["switchyard startup timing:"]
+ for label, stamp in _marks[1:]:
+ lines.append(f" {(stamp - prev) * 1000:8.1f} ms {label}")
+ prev = stamp
+ lines.append(f" {'-' * 8}")
+ lines.append(f" {(prev - start) * 1000:8.1f} ms total (launch invoked -> child spawn)")
+ print("\n".join(lines), file=out)
+ _marks.clear()
diff --git a/tests/test_backend_format_resolver.py b/tests/test_backend_format_resolver.py
index 9ce611434..4d1b1f714 100644
--- a/tests/test_backend_format_resolver.py
+++ b/tests/test_backend_format_resolver.py
@@ -1,10 +1,10 @@
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
-from __future__ import annotations
-
+import httpx
import pytest
+from switchyard.lib import startup_timing
from switchyard.lib.backends import (
backend_format_resolver as resolver_mod,
)
@@ -217,6 +217,67 @@ def test_auto_falls_back_to_openai_when_all_probes_fail(
assert resolution.format is BackendFormat.OPENAI
+def test_auto_chat_completions_timeout_assumes_openai(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ """A Chat Completions probe that times out (raises, not a fast 404) resolves
+ to OPENAI without stacking the slower /v1/messages and /v1/responses probes.
+ A 404 returns False and still falls through — that path is covered by
+ test_auto_resolves_to_anthropic_when_chat_completions_unavailable."""
+ def timed_out(**_: object) -> bool:
+ raise httpx.ReadTimeout("probe timed out")
+
+ monkeypatch.setattr(resolver_mod, "probe_openai_chat_completions_support_sync",
+ timed_out)
+ monkeypatch.setattr(resolver_mod, "probe_anthropic_messages_support_sync",
+ _no_probe("anthropic"))
+ monkeypatch.setattr(resolver_mod, "probe_openai_responses_support_sync",
+ _no_probe("responses"))
+
+ resolution = resolver_mod.BackendFormatResolver.resolve(
+ LlmTarget(
+ model="slow-endpoint-model",
+ format=BackendFormat.AUTO,
+ base_url="https://slow.test/v1",
+ api_key="sk-test", # pragma: allowlist secret
+ timeout_secs=0.05,
+ ),
+ )
+
+ assert resolution.format is BackendFormat.OPENAI
+ assert "timed out" in resolution.reason
+
+
+def test_auto_records_each_probe_in_startup_timing(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ """With timing on, each probe reached leaves its own mark so
+ `launch --startup-timing` can show a per-route breakdown."""
+ monkeypatch.setattr(startup_timing, "enabled", True)
+ startup_timing._marks.clear()
+
+ monkeypatch.setattr(resolver_mod, "probe_openai_chat_completions_support_sync",
+ _RecordingProbe(result=False))
+ monkeypatch.setattr(resolver_mod, "probe_anthropic_messages_support_sync",
+ _RecordingProbe(result=True))
+ monkeypatch.setattr(resolver_mod, "probe_openai_responses_support_sync",
+ _no_probe("responses"))
+
+ resolution = resolver_mod.BackendFormatResolver.resolve(
+ LlmTarget(
+ model="some-model",
+ format=BackendFormat.AUTO,
+ base_url="https://api.anthropic.com/v1",
+ api_key="sk-test", # pragma: allowlist secret
+ ),
+ )
+
+ assert resolution.format is BackendFormat.ANTHROPIC
+ labels = [label for label, _ in startup_timing._marks]
+ assert labels == ["chain init", "probe: /v1/chat/completions", "probe: /v1/messages"]
+ startup_timing._marks.clear()
+
+
# ---------------------------------------------------------------------------
# Missing inputs
# ---------------------------------------------------------------------------