Skip to content

Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.

Rust

fix(litellm): correct seeded routing checks and expand CI coverage (#745)

Graham King35d66831ef11
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 518507d6e..22674ec92 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -166,7 +166,7 @@ jobs:
run: uv run pytest tests/ -v -m "not integration"
litellm:
- name: LiteLLM unit tests
+ name: LiteLLM tests (no live providers)
needs: changes
if: needs.changes.outputs.full_ci == 'true'
runs-on: ubuntu-latest
@@ -188,7 +188,7 @@ jobs:
save-if: ${{ github.ref == 'refs/heads/main' }}
- run: uv sync --locked
- name: pytest
- run: uv run --no-sync pytest tests/unit -v
+ run: uv run --no-sync pytest tests -m "not e2e" -v
# Regression guard: the default install of nemo-switchyard (without extras)
# must NOT drag in torch / transformers / routellm / litellm /
diff --git a/examples/litellm/tests/integration/test_stage_routing.py b/examples/litellm/tests/integration/test_stage_routing.py
index b93d34b4f..2298f8fc4 100644
--- a/examples/litellm/tests/integration/test_stage_routing.py
+++ b/examples/litellm/tests/integration/test_stage_routing.py
@@ -97,22 +97,38 @@ async def test_stage_plugin_controls_real_litellm_router_pipeline() -> None:
assert capable.choices[0].message.content == "served by capable"
-async def test_seeded_random_plugin_reaches_both_litellm_candidates() -> None:
- router = Router(model_list=model_list(), plugins=[RandomRoutingPlugin(seed=6)])
-
- served = {
- (
- await router.acompletion(
+async def test_seeded_random_plugin_repeats_valid_litellm_selections() -> None:
+ sequences = []
+ for _ in range(2):
+ router = Router(model_list=model_list(), plugins=[RandomRoutingPlugin(seed=6)])
+ served = []
+ for index in range(8):
+ response = await router.acompletion(
model=MODEL_GROUP,
messages=[{"role": "user", "content": f"Request {index}"}],
)
+ served.append(response.choices[0].message.content)
+ sequences.append(served)
+
+ assert sequences[0] == sequences[1]
+ assert set(sequences[0]) <= {"served by capable", "served by efficient"}
+
+
+async def test_random_plugin_weights_select_each_litellm_candidate() -> None:
+ for weights, expected in [
+ ([1.0, 0.0], "served by capable"),
+ ([0.0, 1.0], "served by efficient"),
+ ]:
+ router = Router(
+ model_list=model_list(),
+ plugins=[RandomRoutingPlugin(weights=weights, seed=6)],
+ )
+ response = await router.acompletion(
+ model=MODEL_GROUP,
+ messages=[{"role": "user", "content": "Say hello."}],
)
- .choices[0]
- .message.content
- for index in range(2)
- }
- assert served == {"served by capable", "served by efficient"}
+ assert response.choices[0].message.content == expected
class DeploymentRequestRecorder(CustomLogger):