Skip to content

Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.

Rust

chore: Speed up the unit tests (#1203)

Graham Kingbd784e5d4c7b
diff --git a/crates/switchyard-runner/src/config.rs b/crates/switchyard-runner/src/config.rs
index 5982d9417..2e89ab988 100644
--- a/crates/switchyard-runner/src/config.rs
+++ b/crates/switchyard-runner/src/config.rs
@@ -1506,7 +1506,8 @@ new = ["send_message"]
] {
let configured =
source.replace("capable_hold_turns = 2", &format!("{field} = {turns}"));
- let result = runner_from_toml(&configured);
+ let result = AlgorithmFactory::from_toml(&configured)
+ .and_then(|factory| factory.build());
assert_eq!(result.is_ok(), valid, "{field} = {turns}");
if let Err(error) = result {
assert!(error.to_string().contains(field), "{error}");
@@ -2104,7 +2105,11 @@ classifier_magic = true
];
for (toml, expected) in cases {
- let error = error_message(&toml);
+ let error = AlgorithmFactory::from_toml(&toml)
+ .and_then(|factory| factory.build())
+ .err()
+ .expect("invalid configuration should fail")
+ .to_string();
assert!(
error.contains(expected),
"expected error containing {expected}, got {error}"
diff --git a/crates/switchyard-server/Cargo.toml b/crates/switchyard-server/Cargo.toml
index 3601dd083..a6167195c 100644
--- a/crates/switchyard-server/Cargo.toml
+++ b/crates/switchyard-server/Cargo.toml
@@ -54,5 +54,5 @@ async-trait.workspace = true
http-body-util = "0.1"
opentelemetry_sdk = { version = "0.32", default-features = false, features = ["testing"] }
tempfile = "3"
-tokio.workspace = true
+tokio = { workspace = true, features = ["test-util"] }
tower = { version = "0.5", features = ["util"] }
diff --git a/crates/switchyard-server/tests/client_deadline.rs b/crates/switchyard-server/tests/client_deadline.rs
index 5c84c9e1d..a8b498ce4 100644
--- a/crates/switchyard-server/tests/client_deadline.rs
+++ b/crates/switchyard-server/tests/client_deadline.rs
@@ -63,7 +63,7 @@ async fn upstream(
};
if model == "backoff" || (model == "retry-body" && attempt == 1) {
if model == "retry-body" {
- tokio::time::sleep(Duration::from_millis(300)).await;
+ tokio::time::sleep(Duration::from_millis(100)).await;
}
let delay = if model == "backoff" { "1" } else { "0" };
return (
@@ -105,7 +105,7 @@ async fn upstream(
"choices":[{"index":0,"message":{"role":"assistant","content":content},"finish_reason":"stop"}],
"usage":{"prompt_tokens":3,"completion_tokens":1,"total_tokens":4}});
if model == "slow-body" || model == "retry-body" {
- let delay = if model == "retry-body" { 450 } else { 3000 };
+ let delay = if model == "retry-body" { 150 } else { 3000 };
let stream = async_stream::stream! {
yield Ok::<_, Infallible>(Bytes::from_static(b" "));
tokio::time::sleep(Duration::from_millis(delay)).await;
@@ -297,15 +297,15 @@ async fn client_deadline_stops_routing_and_counts_attempts() -> TestResult {
for (judge, expected_calls, counts) in [
("slow-body", &["slow-body"][..], [0., 0., 1.]),
("backoff", &["backoff"][..], [0., 1., 0.]),
- // The first attempt waits 300 ms and the retry body waits 450 ms.
- // Each fits the 600 ms deadline alone; together they exceed it.
+ // The first attempt waits 100 ms and the retry body waits 150 ms.
+ // Each fits the 200 ms deadline alone; together they exceed it.
(
"retry-body",
&["retry-body", "retry-body"][..],
[0., 1., 1.],
),
] {
- let app = app(&upstream, "classifier", judge, "weak", 600)?;
+ let app = app(&upstream, "classifier", judge, "weak", 200)?;
let (status, body) = check(
&upstream,
&app,
@@ -315,7 +315,7 @@ async fn client_deadline_stops_routing_and_counts_attempts() -> TestResult {
)
.await?;
assert_eq!(status, StatusCode::GATEWAY_TIMEOUT, "{judge}: {body}");
- assert!(body.contains("600 ms"), "{body}");
+ assert!(body.contains("200 ms"), "{body}");
}
let failed_stream = app(&upstream, "classifier", "stream-error", "weak", 100)?;
let (status, body) = check(
diff --git a/crates/switchyard-server/tests/server.rs b/crates/switchyard-server/tests/server.rs
index 501b971e2..5fb4c6f24 100644
--- a/crates/switchyard-server/tests/server.rs
+++ b/crates/switchyard-server/tests/server.rs
@@ -153,6 +153,7 @@ async fn upstream_chat(
} else {
StatusCode::TOO_MANY_REQUESTS
},
+ [("retry-after", "0")],
Json(json!({"error": {"message": "quota exhausted", "code": "usage_limit"}})),
)
.into_response();
@@ -1695,7 +1696,8 @@ impl Response {
async fn models_probes_routes_concurrently_and_caches_failed_discovery() -> TestResult {
for stalled in [false, true] {
let calls = Arc::new(Mutex::new(Vec::new()));
- let barrier = Arc::new(tokio::sync::Barrier::new(3));
+ let barrier = Arc::new(tokio::sync::Barrier::new(4));
+ let probe_barrier = Arc::clone(&barrier);
let captured = Arc::clone(&calls);
let upstream_app = Router::new().route(
"/props",
@@ -1752,13 +1754,23 @@ llm_client = "efficient"
upstream.base_url
))?);
- let (first, concurrent) = tokio::time::timeout(std::time::Duration::from_secs(3), async {
- tokio::join!(
- send(&app, "GET", "/v1/models", None),
- send(&app, "GET", "/v1/models", None)
- )
- })
- .await?;
+ let (first, concurrent, ()) =
+ tokio::time::timeout(std::time::Duration::from_secs(3), async {
+ tokio::join!(
+ send(&app, "GET", "/v1/models", None),
+ send(&app, "GET", "/v1/models", None),
+ async {
+ probe_barrier.wait().await;
+ if stalled {
+ // All probes have reached the server and started their deadlines.
+ tokio::time::pause();
+ tokio::time::advance(std::time::Duration::from_secs(2)).await;
+ tokio::time::resume();
+ }
+ }
+ )
+ })
+ .await?;
let models = first?.json()?;
assert_eq!(concurrent?.json()?, models);
assert_eq!(models["data"].as_array().unwrap().len(), 3);