Skip to content

Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.

Rust

fix: Missed some LiteLLM plugin python in API migration (#699)

Graham King739181ae423e
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 4186d3416..4c81b7493 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -165,6 +165,31 @@ jobs:
- name: pytest
run: uv run pytest tests/ -v -m "not integration"
+ litellm:
+ name: LiteLLM unit tests
+ needs: changes
+ if: needs.changes.outputs.full_ci == 'true'
+ runs-on: ubuntu-latest
+ defaults:
+ run:
+ working-directory: examples/litellm
+ env:
+ LITELLM_LOCAL_MODEL_COST_MAP: "True"
+ steps:
+ - uses: actions/checkout@v4
+ - uses: astral-sh/setup-uv@v6
+ with:
+ python-version: "3.12"
+ enable-cache: true
+ cache-dependency-glob: "examples/litellm/uv.lock"
+ - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1
+ with:
+ cache-bin: false
+ save-if: ${{ github.ref == 'refs/heads/main' }}
+ - run: uv sync --locked
+ - name: pytest
+ run: uv run --no-sync pytest tests/unit -v
+
# Regression guard: the default install of nemo-switchyard (without extras)
# must NOT drag in torch / transformers / routellm / litellm /
# openai-agents / docx. Catches accidental top-level imports of
@@ -227,7 +252,7 @@ jobs:
ci-success:
name: CI Success
if: always()
- needs: [changes, lint, spdx-headers, rust, typecheck, test, slim-install-smoke]
+ needs: [changes, lint, spdx-headers, rust, typecheck, test, litellm, slim-install-smoke]
runs-on: ubuntu-latest
steps:
- name: Verify all jobs succeeded
diff --git a/examples/litellm/deployment/Dockerfile b/examples/litellm/deployment/Dockerfile
index ade4cd67a..d3afadd7d 100644
--- a/examples/litellm/deployment/Dockerfile
+++ b/examples/litellm/deployment/Dockerfile
@@ -33,7 +33,7 @@ RUN python -c "from importlib.metadata import version; \
from switchyard.libsy import algorithms; \
from switchyard_litellm import SwitchyardRoutingPlugin; \
assert version('litellm') == '1.97.0'; \
-SwitchyardRoutingPlugin(algorithms.random(['smoke']))"
+SwitchyardRoutingPlugin(algorithms.random())"
RUN addgroup -S -g 10001 litellm \
&& adduser -S -D -u 10001 -G litellm -h /home/litellm litellm
diff --git a/examples/litellm/src/switchyard_litellm/plugins/random_routing_plugin.py b/examples/litellm/src/switchyard_litellm/plugins/random_routing_plugin.py
index 6e6435271..6064813c8 100644
--- a/examples/litellm/src/switchyard_litellm/plugins/random_routing_plugin.py
+++ b/examples/litellm/src/switchyard_litellm/plugins/random_routing_plugin.py
@@ -37,7 +37,8 @@ class RandomRoutingPlugin(LiteLLMRequestRewriter):
plugin = self._plugins.get(candidates)
if plugin is None:
plugin = SwitchyardRoutingPlugin(
- algorithms.random(candidates, weights=self._weights, seed=self._seed)
+ algorithms.random(weights=self._weights, seed=self._seed),
+ models={"any": candidates},
)
self._plugins[candidates] = plugin
return await plugin.run(context)
diff --git a/examples/litellm/tests/unit/test_candidate_routing_plugins.py b/examples/litellm/tests/unit/test_candidate_routing_plugins.py
index 0f1cfa1e5..b5e48c5e8 100644
--- a/examples/litellm/tests/unit/test_candidate_routing_plugins.py
+++ b/examples/litellm/tests/unit/test_candidate_routing_plugins.py
@@ -86,24 +86,22 @@ async def test_seeded_random_uses_and_retains_the_full_deployer_candidate_pool()
assert plugin_type is not None, "RandomRoutingPlugin must be part of the public integration API"
plugin = plugin_type(seed=6)
assert isinstance(plugin, CustomLogger)
+ replay_plugin = plugin_type(seed=6)
candidates = ["provider/alpha", "provider/beta", "provider/gamma"]
selected: list[str] = []
- for index in range(5):
- context = routing_context(
- [{"role": "user", "content": f"Request {index}."}],
- candidates.copy(),
- )
+ for index in range(100):
+ messages = [{"role": "user", "content": f"Request {index}."}]
+ context = routing_context(messages, candidates.copy())
+ replay_context = routing_context(messages, candidates.copy())
await plugin.run(context)
+ await replay_plugin.run(replay_context)
+
+ assert len(context.candidate_models) == 1
+ assert context.candidate_models == replay_context.candidate_models
selected.extend(context.candidate_models)
- assert selected == [
- "provider/beta",
- "provider/gamma",
- "provider/gamma",
- "provider/gamma",
- "provider/alpha",
- ]
+ assert set(selected) == set(candidates)
async def test_random_rejects_an_empty_candidate_pool() -> None:
@@ -114,8 +112,17 @@ async def test_random_rejects_an_empty_candidate_pool() -> None:
await plugin.run(context)
-async def test_random_weights_apply_to_unique_candidate_models() -> None:
- plugin = switchyard_litellm.RandomRoutingPlugin(weights=[0.0, 1.0], seed=6)
+ ("weights", "expected"),
+ [
+ ([0.0, 1.0], ["provider/beta"]),
+ ([1.0, 0.0], ["provider/alpha", "provider/alpha"]),
+ ],
+)
+async def test_random_weights_apply_to_unique_candidate_models(
+ weights: list[float], expected: list[str]
+) -> None:
+ plugin = switchyard_litellm.RandomRoutingPlugin(weights=weights, seed=6)
context = routing_context(
[{"role": "user", "content": "Hello."}],
["provider/alpha", "provider/alpha", "provider/beta"],
@@ -123,4 +130,21 @@ async def test_random_weights_apply_to_unique_candidate_models() -> None:
await plugin.run(context)
- assert context.candidate_models == ["provider/beta"]
+ assert context.candidate_models == expected
+
+
+async def test_random_uses_the_current_candidate_pool() -> None:
+ plugin = switchyard_litellm.RandomRoutingPlugin(seed=6)
+ pools = [
+ ["provider/only"],
+ ["provider/alpha", "provider/beta"],
+ ["provider/other"],
+ ["provider/only"],
+ ]
+ for candidates in pools:
+ context = routing_context([{"role": "user", "content": "Hello."}], candidates.copy())
+
+ await plugin.run(context)
+
+ assert len(context.candidate_models) == 1
+ assert context.candidate_models[0] in candidates