Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.
Rust
fix: Missed some LiteLLM plugin python in API migration (#699)
Graham King739181ae423e
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.ymlindex 4186d3416..4c81b7493 100644--- a/.github/workflows/ci.yml+++ b/.github/workflows/ci.yml@@ -165,6 +165,31 @@ jobs:- name: pytestrun: uv run pytest tests/ -v -m "not integration"+ litellm:+ name: LiteLLM unit tests+ needs: changes+ if: needs.changes.outputs.full_ci == 'true'+ runs-on: ubuntu-latest+ defaults:+ run:+ working-directory: examples/litellm+ env:+ LITELLM_LOCAL_MODEL_COST_MAP: "True"+ steps:+ - uses: actions/checkout@v4+ - uses: astral-sh/setup-uv@v6+ with:+ python-version: "3.12"+ enable-cache: true+ cache-dependency-glob: "examples/litellm/uv.lock"+ - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1+ with:+ cache-bin: false+ save-if: ${{ github.ref == 'refs/heads/main' }}+ - run: uv sync --locked+ - name: pytest+ run: uv run --no-sync pytest tests/unit -v+# Regression guard: the default install of nemo-switchyard (without extras)# must NOT drag in torch / transformers / routellm / litellm /# openai-agents / docx. Catches accidental top-level imports of@@ -227,7 +252,7 @@ jobs:ci-success:name: CI Successif: always()- needs: [changes, lint, spdx-headers, rust, typecheck, test, slim-install-smoke]+ needs: [changes, lint, spdx-headers, rust, typecheck, test, litellm, slim-install-smoke]runs-on: ubuntu-lateststeps:- name: Verify all jobs succeededdiff --git a/examples/litellm/deployment/Dockerfile b/examples/litellm/deployment/Dockerfileindex ade4cd67a..d3afadd7d 100644--- a/examples/litellm/deployment/Dockerfile+++ b/examples/litellm/deployment/Dockerfile@@ -33,7 +33,7 @@ RUN python -c "from importlib.metadata import version; \from switchyard.libsy import algorithms; \from switchyard_litellm import SwitchyardRoutingPlugin; \assert version('litellm') == '1.97.0'; \-SwitchyardRoutingPlugin(algorithms.random(['smoke']))"+SwitchyardRoutingPlugin(algorithms.random())"RUN addgroup -S -g 10001 litellm \&& adduser -S -D -u 10001 -G litellm -h /home/litellm litellmdiff --git a/examples/litellm/src/switchyard_litellm/plugins/random_routing_plugin.py b/examples/litellm/src/switchyard_litellm/plugins/random_routing_plugin.pyindex 6e6435271..6064813c8 100644--- a/examples/litellm/src/switchyard_litellm/plugins/random_routing_plugin.py+++ b/examples/litellm/src/switchyard_litellm/plugins/random_routing_plugin.py@@ -37,7 +37,8 @@ class RandomRoutingPlugin(LiteLLMRequestRewriter):plugin = self._plugins.get(candidates)if plugin is None:plugin = SwitchyardRoutingPlugin(- algorithms.random(candidates, weights=self._weights, seed=self._seed)+ algorithms.random(weights=self._weights, seed=self._seed),+ models={"any": candidates},)self._plugins[candidates] = pluginreturn await plugin.run(context)diff --git a/examples/litellm/tests/unit/test_candidate_routing_plugins.py b/examples/litellm/tests/unit/test_candidate_routing_plugins.pyindex 0f1cfa1e5..b5e48c5e8 100644--- a/examples/litellm/tests/unit/test_candidate_routing_plugins.py+++ b/examples/litellm/tests/unit/test_candidate_routing_plugins.py@@ -86,24 +86,22 @@ async def test_seeded_random_uses_and_retains_the_full_deployer_candidate_pool()assert plugin_type is not None, "RandomRoutingPlugin must be part of the public integration API"plugin = plugin_type(seed=6)assert isinstance(plugin, CustomLogger)+ replay_plugin = plugin_type(seed=6)candidates = ["provider/alpha", "provider/beta", "provider/gamma"]selected: list[str] = []- for index in range(5):- context = routing_context(- [{"role": "user", "content": f"Request {index}."}],- candidates.copy(),- )+ for index in range(100):+ messages = [{"role": "user", "content": f"Request {index}."}]+ context = routing_context(messages, candidates.copy())+ replay_context = routing_context(messages, candidates.copy())await plugin.run(context)+ await replay_plugin.run(replay_context)++ assert len(context.candidate_models) == 1+ assert context.candidate_models == replay_context.candidate_modelsselected.extend(context.candidate_models)- assert selected == [- "provider/beta",- "provider/gamma",- "provider/gamma",- "provider/gamma",- "provider/alpha",- ]+ assert set(selected) == set(candidates)async def test_random_rejects_an_empty_candidate_pool() -> None:@@ -114,8 +112,17 @@ async def test_random_rejects_an_empty_candidate_pool() -> None:await plugin.run(context)-async def test_random_weights_apply_to_unique_candidate_models() -> None:- plugin = switchyard_litellm.RandomRoutingPlugin(weights=[0.0, 1.0], seed=6)+ ("weights", "expected"),+ [+ ([0.0, 1.0], ["provider/beta"]),+ ([1.0, 0.0], ["provider/alpha", "provider/alpha"]),+ ],+)+async def test_random_weights_apply_to_unique_candidate_models(+ weights: list[float], expected: list[str]+) -> None:+ plugin = switchyard_litellm.RandomRoutingPlugin(weights=weights, seed=6)context = routing_context([{"role": "user", "content": "Hello."}],["provider/alpha", "provider/alpha", "provider/beta"],@@ -123,4 +130,21 @@ async def test_random_weights_apply_to_unique_candidate_models() -> None:await plugin.run(context)- assert context.candidate_models == ["provider/beta"]+ assert context.candidate_models == expected+++async def test_random_uses_the_current_candidate_pool() -> None:+ plugin = switchyard_litellm.RandomRoutingPlugin(seed=6)+ pools = [+ ["provider/only"],+ ["provider/alpha", "provider/beta"],+ ["provider/other"],+ ["provider/only"],+ ]+ for candidates in pools:+ context = routing_context([{"role": "user", "content": "Hello."}], candidates.copy())++ await plugin.run(context)++ assert len(context.candidate_models) == 1+ assert context.candidate_models[0] in candidates