Skip to content

Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.

Rust

feat(ffi): add asynchronous routing handles (#1077)

Alex Fournierb0fa4fb201e5
diff --git a/Cargo.lock b/Cargo.lock
index e7f816931..a9ae848f0 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -5415,10 +5415,16 @@ dependencies = [
name = "switchyard-ffi"
version = "0.3.0"
dependencies = [
+ "futures",
+ "parking_lot",
"serde",
"serde_json",
+ "switchyard-libsy",
"switchyard-protocol",
+ "switchyard-runner",
"switchyard-translation",
+ "tokio",
+ "tokio-util",
]
[[package]]
diff --git a/crates/switchyard-ffi/Cargo.toml b/crates/switchyard-ffi/Cargo.toml
index c113438ed..96e48d3b9 100644
--- a/crates/switchyard-ffi/Cargo.toml
+++ b/crates/switchyard-ffi/Cargo.toml
@@ -13,7 +13,13 @@ publish = false
crate-type = ["cdylib", "staticlib"]
[dependencies]
+futures.workspace = true
+parking_lot.workspace = true
serde.workspace = true
serde_json.workspace = true
+switchyard-libsy.workspace = true
switchyard-protocol.workspace = true
+switchyard-runner.workspace = true
switchyard-translation.workspace = true
+tokio.workspace = true
+tokio-util = { version = "0.7", features = ["rt"] }
diff --git a/crates/switchyard-ffi/README.md b/crates/switchyard-ffi/README.md
index 4b1ec9866..9a20b330d 100644
--- a/crates/switchyard-ffi/README.md
+++ b/crates/switchyard-ffi/README.md
@@ -1,21 +1,43 @@
-# C translation interface
+# C routing interface
-An experimental C interface to Switchyard's request and response codecs. The
-interface uses NUL-terminated JSON strings. Ownership rules are in
+An experimental C interface to libsy. The host supplies eligible models and
+executes model and decision calls. The interface uses opaque handles and
+NUL-terminated JSON strings. Ownership and concurrency rules are in
[`include/switchyard.h`](include/switchyard.h).
## JSON payloads
| Function | Input |
| --- | --- |
+| `sy_router_new` | `{algorithm, targets}`: an `AlgorithmSpec` and target-alias-to-model-ID map |
+| `sy_run_new` | `{request, models, subagent_models?, metadata?}` |
+| `sy_run_respond` | `{response, served_model?}` or `{error}` |
| `sy_decode_request`, `sy_decode_response` | `{format, body}` |
| `sy_encode_request` | `{format, request, model}` |
`request` and model responses use Switchyard's normalized protocol JSON.
-`sy_encode_request` applies the selected model before encoding the request.
+`models` and `subagent_models` map category names to ordered model-ID lists.
+Optional metadata fields are `session_id`, `agent_id`, `is_subagent`,
+`is_delegated_work`, and `session_final`.
+
+For model replies, `served_model` identifies the target that answered, which
+can differ from both the first candidate and the provider's response model name.
+Decision replies contain a normalized `DecisionResponse`; `served_model` does
+not apply to them. Error replies contain a string.
+
+Each accepted read delivers one step or an error through its callback:
+
+- `call_model`: `{type, id, models, request}`
+- `call_decision`: `{type, id, model, request}`
+- `done`: `{type, selected_model_ids, request, fallback_request, response, metadata}`
+
+A callback error or `done` ends the run. Outcomes retain ordered targets, the
+prepared request, an optional buffered answer, and optional routing evidence.
+The host executes the answer call when `response` is null.
+Streamed routing answers are collected into a buffered response before `done`.
+Cancellation interrupts collection; stream errors end the run with a callback error.
+When `fallback_request` is non-null, use it for candidates after the first.
+Content-policy denials must not advance to those fallback candidates.
Wire formats are `openai_chat`, `openai_responses`, and `anthropic_messages`.
The translation functions use the existing Switchyard codecs.
-
-Fallible calls return NULL on success or an owned error string. Free every
-returned error or JSON string with `sy_string_free`.
diff --git a/crates/switchyard-ffi/include/switchyard.h b/crates/switchyard-ffi/include/switchyard.h
index ded662702..3ff8fe08a 100644
--- a/crates/switchyard-ffi/include/switchyard.h
+++ b/crates/switchyard-ffi/include/switchyard.h
@@ -10,13 +10,34 @@
#define SY_ABI_VERSION 1
uint32_t sy_abi_version(void);
+typedef struct SyRouter SyRouter;
+typedef struct SyRun SyRun;
+typedef void (*SyNextCallback)(uintptr_t user_data, char *error, char *step_json);
+
// Fallible calls return NULL on success or an owned error string.
// JSON inputs are borrowed for the call and must be NUL-terminated.
-// Free all output strings with sy_string_free.
+// Free all output strings, including callback strings, with sy_string_free.
+char *sy_router_new(const char *config_json, SyRouter **out);
+char *sy_run_new(const SyRouter *router, const char *input_json, SyRun **out);
+
+// One pending read per run; other runs execute independently. Replies and cancellation may overlap.
+// NULL accepts one read (one callback, even if cancelled); errors reject it (no callback).
+// Callbacks may overlap or run before return. Keep callback state alive until they return.
+// Callbacks must return promptly, not unwind, and not call sy_run_respond or free handles.
+char *sy_run_next_async(const SyRun *run, SyNextCallback callback, uintptr_t user_data);
+char *sy_run_respond(const SyRun *run, uint64_t call_id, const char *reply_json);
+
char *sy_decode_request(const char *input_json, char **out_json);
char *sy_encode_request(const char *input_json, char **out_json);
char *sy_decode_response(const char *input_json, char **out_json);
+// Signals cancellation without waiting for callbacks.
+void sy_run_cancel(const SyRun *run);
+// Free a run once, after other calls using it return; accepted callbacks may still be active.
+// Cancels and waits for those callbacks. Do not call from a callback or Tokio task.
+void sy_run_free(SyRun *run);
+// Free a router once, with no calls using it. Existing runs retain their engine.
+void sy_router_free(SyRouter *router);
void sy_string_free(char *string);
#endif
diff --git a/crates/switchyard-ffi/src/lib.rs b/crates/switchyard-ffi/src/lib.rs
index 2a0c73a09..b004b5ecd 100644
--- a/crates/switchyard-ffi/src/lib.rs
+++ b/crates/switchyard-ffi/src/lib.rs
@@ -1,15 +1,28 @@
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0
-//! C bridge to Switchyard's request and response codecs.
+//! Pull-based C bridge to libsy.
+use std::collections::{BTreeMap, HashMap};
use std::ffi::{CStr, CString, c_char};
use std::panic::{AssertUnwindSafe, catch_unwind};
+use std::sync::Arc;
+use futures::{FutureExt, StreamExt};
+use parking_lot::Mutex;
use serde::{Deserialize, Serialize};
-use switchyard_protocol::{LlmRequest, ModelId, WireFormat};
+use switchyard_libsy::{Algorithm, Call, LibsyError, RuntimeModels, State, Step, StepStream};
+use switchyard_protocol::{
+ AggLlmResponse, Category, DecisionRequest, LlmRequest, LlmResponse, Metadata, ModelId, Request,
+ Response, WireFormat,
+};
+use switchyard_runner::AlgorithmSpec;
+use tokio::runtime::Runtime;
+use tokio_util::sync::CancellationToken;
+use tokio_util::task::TaskTracker;
type Result<T> = std::result::Result<T, Box<dyn std::error::Error>>;
+type ModelGroups = HashMap<String, Option<Vec<ModelId>>>;
/// Returns the compatibility generation for C signatures and JSON payloads.
#[unsafe(no_mangle)]
@@ -17,6 +30,31 @@ pub extern "C" fn sy_abi_version() -> u32 {
1
}
+#[derive(Deserialize)]
+struct RouterConfig {
+ algorithm: AlgorithmSpec,
+ targets: BTreeMap<String, ModelId>,
+}
+
+#[derive(Deserialize)]
+struct RunInput {
+ request: LlmRequest,
+ models: ModelGroups,
+ subagent_models: Option<ModelGroups>,
+ metadata: Option<RequestMetadata>,
+}
+
+#[derive(Deserialize)]
+struct RequestMetadata {
+ session_id: Option<String>,
+ agent_id: Option<String>,
+ #[serde(default)]
+ is_subagent: bool,
+ #[serde(default)]
+ is_delegated_work: bool,
+ session_final: Option<bool>,
+}
+
#[derive(Deserialize)]
struct DecodeInput {
format: WireFormat,
@@ -30,6 +68,254 @@ struct EncodeInput {
model: ModelId,
}
+#[derive(Deserialize)]
+#[serde(untagged)]
+enum CallReply {
+ Error {
+ error: String,
+ },
+ Response {
+ response: serde_json::Value,
+ served_model: Option<ModelId>,
+ },
+}
+
+#[derive(Serialize)]
+struct OutcomeMetadataRef<'a> {
+ outcome_id: &'a str,
+ algorithm: &'a str,
+ evidence: Option<&'a serde_json::Value>,
+}
+
+#[derive(Serialize)]
+#[serde(tag = "type", rename_all = "snake_case")]
+enum RunStep<'a> {
+ CallModel {
+ id: u64,
+ models: &'a [ModelId],
+ request: &'a LlmRequest,
+ },
+ CallDecision {
+ id: u64,
+ model: &'a ModelId,
+ request: &'a DecisionRequest,
+ },
+ Done {
+ selected_model_ids: &'a [ModelId],
+ request: &'a LlmRequest,
+ fallback_request: Option<&'a LlmRequest>,
+ response: Option<&'a AggLlmResponse>,
+ metadata: Option<OutcomeMetadataRef<'a>>,
+ },
+}
+
+struct Engine {
+ algorithm: Arc<dyn Algorithm>,
+ runtime: Runtime,
+}
+
+/// Algorithm handle; routing runs retain its engine.
+pub struct SyRouter {
+ engine: Arc<Engine>,
+}
+
+struct Reader {
+ stream: Option<StepStream>,
+ next_id: u64,
+}
+
+enum RunStatus {
+ Active,
+ Completed,
+ Cancelled,
+ Failed,
+}
+
+impl RunStatus {
+ fn require_active(&self) -> Result<()> {
+ match self {
+ Self::Active => Ok(()),
+ Self::Completed => Err("routing already completed".into()),
+ Self::Cancelled => Err("routing cancelled".into()),
+ Self::Failed => Err("routing failed".into()),
+ }
+ }
+}
+
+struct Lifecycle {
+ status: RunStatus,
+ is_reading: bool,
+ pending: HashMap<u64, Call>,
+}
+
+struct RunState {
+ reader: tokio::sync::Mutex<Reader>,
+ lifecycle: Mutex<Lifecycle>,
+ cancellation: CancellationToken,
+}
+
+/// Routing run that owns pending calls and callback cleanup.
+pub struct SyRun {
+ state: Arc<RunState>,
+ tasks: TaskTracker,
+ // Tasks retain only run state; free joins them before dropping the runtime.
+ engine: Arc<Engine>,
+}
+
+impl SyRun {
+ fn new(stream: StepStream, engine: Arc<Engine>) -> Self {
+ Self {
+ state: Arc::new(RunState {
+ reader: tokio::sync::Mutex::new(Reader {
+ stream: Some(stream),
+ next_id: 1,
+ }),
+ lifecycle: Mutex::new(Lifecycle {
+ status: RunStatus::Active,
+ is_reading: false,
+ pending: HashMap::new(),
+ }),
+ cancellation: CancellationToken::new(),
+ }),
+ tasks: TaskTracker::new(),
+ engine,
+ }
+ }
+}
+
+impl Drop for SyRun {
+ fn drop(&mut self) {
+ self.state.stop(RunStatus::Cancelled);
+ self.tasks.close();
+ self.engine.runtime.block_on(self.tasks.wait());
+ }
+}
+
+struct ReadGuard {
+ state: Arc<RunState>,
+}
+
+impl Drop for ReadGuard {
+ fn drop(&mut self) {
+ self.state.lifecycle.lock().is_reading = false;
+ }
+}
+
+impl RunState {
+ fn begin_read(self: &Arc<Self>) -> Result<ReadGuard> {
+ let mut lifecycle = self.lifecycle.lock();
+ lifecycle.status.require_active()?;
+ if lifecycle.is_reading {
+ return Err("routing read already in progress".into());
+ }
+ lifecycle.is_reading = true;
+ Ok(ReadGuard {
+ state: Arc::clone(self),
+ })
+ }
+
+ // Serialize cancellation with reply delivery.
+ fn stop(&self, status: RunStatus) {
+ {
+ let mut lifecycle = self.lifecycle.lock();
+ if matches!(lifecycle.status, RunStatus::Active) {
+ lifecycle.status = status;
+ }
+ lifecycle.pending.clear();
+ }
+ self.cancellation.cancel();
+ if let Ok(mut reader) = self.reader.try_lock() {
+ reader.stream.take();
+ }
+ }
+
+ async fn next_step(&self, reader: &mut Reader) -> Result<CString> {
+ self.lifecycle.lock().status.require_active()?;
+ let stream = reader.stream.as_mut().ok_or("routing stream is closed")?;
+ let step = tokio::select! {
+ biased;
+ _ = self.cancellation.cancelled() => None,
+ step = stream.next() => step,
+ };
+ self.lifecycle.lock().status.require_active()?;
+ let step = step.ok_or("routing stream ended without an outcome")??;
+ let id = reader.next_id;
+ reader.next_id += 1;
+ match step {
+ Step::CallModel(call) => {
+ let output = encode_json(&RunStep::CallModel {
+ id,
+ models: &call.models,
+ request: &call.request.llm_request,
+ })?;
+ let mut lifecycle = self.lifecycle.lock();
+ lifecycle.status.require_active()?;
+ lifecycle.pending.insert(id, Call::Model(call));
+ Ok(output)
+ }
+ Step::CallDecision(call) => {
+ let output = encode_json(&RunStep::CallDecision {
+ id,
+ model: &call.model,
+ request: &call.request,
+ })?;
+ let mut lifecycle = self.lifecycle.lock();
+ lifecycle.status.require_active()?;
+ lifecycle.pending.insert(id, Call::Decision(call));
+ Ok(output)
+ }
+ Step::Done(outcome) => {
+ let response = match outcome.response {
+ None => None,
+ Some(response) => tokio::select! {
+ biased;
+ _ = self.cancellation.cancelled() => return Err("routing cancelled".into()),
+ response = response.llm_response.into_agg() => Some(response?),
+ },
+ };
+ let output = encode_json(&RunStep::Done {
+ selected_model_ids: &outcome.selected_model_ids,
+ request: &outcome.request.llm_request,
+ fallback_request: outcome
+ .fallback_request
+ .as_ref()
+ .map(|request| &request.llm_request),
+ response: response.as_ref(),
+ metadata: outcome
+ .metadata
+ .as_ref()
+ .map(|metadata| OutcomeMetadataRef {
+ outcome_id: metadata.outcome_id(),
+ algorithm: &metadata.algorithm,
+ evidence: metadata.evidence.as_ref(),
+ }),
+ })?;
+ let mut lifecycle = self.lifecycle.lock();
+ lifecycle.status.require_active()?;
+ lifecycle.status = RunStatus::Completed;
+ lifecycle.pending.clear();
+ reader.stream.take();
+ Ok(output)
+ }
+ }
+ }
+}
+
+impl ReadGuard {
+ async fn next(self) -> Result<CString> {
+ let mut reader = self.state.reader.lock().await;
+ let result = AssertUnwindSafe(self.state.next_step(&mut reader))
+ .catch_unwind()
+ .await
+ .unwrap_or_else(|_| Err("Switchyard binding panicked".into()));
+ if result.is_err() {
+ self.state.stop(RunStatus::Failed);
+ reader.stream.take();
+ }
+ result
+ }
+}
+
fn error_string(message: String) -> *mut c_char {
match CString::new(message) {
Ok(message) => message.into_raw(),
@@ -125,6 +411,225 @@ pub unsafe extern "C" fn sy_decode_response(
})
}
+fn parse_model_groups(groups: ModelGroups) -> Result<HashMap<Category, Vec<ModelId>>> {
+ groups
+ .into_iter()
+ .map(|(name, models)| Ok((name.parse()?, models.unwrap_or_default())))
+ .collect()
+}
+
+/// Builds an algorithm without provider clients.
+/// # Safety
+/// JSON must be NUL-terminated and `out` writable. Free the returned handle with `sy_router_free`.
+#[unsafe(no_mangle)]
+pub unsafe extern "C" fn sy_router_new(
+ config_json: *const c_char,
+ out: *mut *mut SyRouter,
+) -> *mut c_char {
+ ffi_call(|| {
+ let out = unsafe { out.as_mut() }.ok_or("missing router output")?;
+ *out = std::ptr::null_mut();
+ let config: RouterConfig = unsafe { parse_json_input(config_json) }?;
+ let algorithm = config.algorithm.build("embedded", &config.targets)?;
+ let runtime = Runtime::new()?;
+ *out = Box::into_raw(Box::new(SyRouter {
+ engine: Arc::new(Engine { algorithm, runtime }),
+ }));
+ Ok(())
+ })
+}
+
+/// Starts a run with host-approved model groups.
+/// # Safety
+/// Handles must be live, JSON NUL-terminated, and `out` writable.
+#[unsafe(no_mangle)]
+pub unsafe extern "C" fn sy_run_new(
+ router: *const SyRouter,
+ input_json: *const c_char,
+ out: *mut *mut SyRun,
+) -> *mut c_char {
+ ffi_call(|| {
+ let router = unsafe { router.as_ref() }.ok_or("missing router")?;
+ let out = unsafe { out.as_mut() }.ok_or("missing run output")?;
+ *out = std::ptr::null_mut();
+ let input: RunInput = unsafe { parse_json_input(input_json) }?;
+ let mut models = RuntimeModels::new(parse_model_groups(input.models)?);
+ if let Some(subagent_models) = input.subagent_models {
+ models = models.with_subagent(parse_model_groups(subagent_models)?);
+ }
+ let request = Request {
+ llm_request: input.request,
+ metadata: input.metadata.map(|metadata| Metadata {
+ session_id: metadata.session_id,
+ agent_id: metadata.agent_id,
+ is_subagent: metadata.is_subagent,
+ is_delegated_work: metadata.is_delegated_work,
+ session_final: metadata.session_final,
+ ..Default::default()
+ }),
+ ..Default::default()
+ };
+ let _entered = router.engine.runtime.enter();
+ let stream = Arc::clone(&router.engine.algorithm).run_stream(
+ request,
+ Arc::new(models),
+ State::default(),
+ );
+ *out = Box::into_raw(Box::new(SyRun::new(stream, Arc::clone(&router.engine))));
+ Ok(())
+ })
+}
+
+/// Requests the next step asynchronously. Only one read may be pending per run.
+/// NULL accepts the read (one callback); an owned error rejects it (no callback).
+/// Other runs, model replies, and cancellation can proceed independently.
+/// # Safety
+/// Keep the run live during this call and the callback state live until its callback returns.
+/// Callbacks may overlap or run before this call returns. Free their strings with `sy_string_free`.
+/// Callbacks must return promptly without unwinding, calling `sy_run_respond`, or freeing handles.
+#[unsafe(no_mangle)]
+pub unsafe extern "C" fn sy_run_next_async(
+ run: *const SyRun,
+ callback: Option<unsafe extern "C" fn(usize, *mut c_char, *mut c_char)>,
+ user_data: usize,
+) -> *mut c_char {
+ ffi_call(|| {
+ let run = unsafe { run.as_ref() }.ok_or("missing run")?;
+ let callback = callback.ok_or("missing completion callback")?;
+ let read = run.state.begin_read()?;
+ run.tasks.spawn_on(
+ async move {
+ let (error, output) = match read.next().await {
+ Ok(output) => (std::ptr::null_mut(), output.into_raw()),
+ Err(error) => (error_string(error.to_string()), std::ptr::null_mut()),
+ };
+ unsafe { callback(user_data, error, output) };
+ },
+ run.engine.runtime.handle(),
+ );
+ Ok(())
+ })
+}
+
+/// Delivers a buffered host reply to its pending call.
+/// # Safety
+/// The run must remain live and JSON must be NUL-terminated. May overlap an async read.
+/// Do not call from a completion callback or Tokio task.
+#[unsafe(no_mangle)]
+pub unsafe extern "C" fn sy_run_respond(
+ run: *const SyRun,
+ call_id: u64,
+ reply_json: *const c_char,
+) -> *mut c_char {
+ ffi_call(|| {
+ let run = unsafe { run.as_ref() }.ok_or("missing run")?;
+ let reply: CallReply = unsafe { parse_json_input(reply_json) }?;
+ let mut lifecycle = run.state.lifecycle.lock();
+ lifecycle.status.require_active()?;
+ let call = lifecycle
+ .pending
+ .remove(&call_id)
+ .ok_or("unknown call ID")?;
+ let result = match call {
+ Call::Model(call) => {
+ let response = match reply {
+ CallReply::Error { error } => Ok(Err(LibsyError::external(
+ "host model call",
+ std::io::Error::other(error),
+ ))),
+ CallReply::Response {
+ response,
+ served_model,
+ } => serde_json::from_value(response).map(|response| {
+ Ok(Response {
+ llm_response: LlmResponse::Agg(response),
+ metadata: served_model.map(|model| Metadata {
+ served_model: Some(model),
+ ..Default::default()
+ }),
+ upstream_headers: Default::default(),
+ })
+ }),
+ };
+ let response = match response {
+ Ok(response) => response,
+ Err(error) => {
+ lifecycle.pending.insert(call_id, Call::Model(call));
+ return Err(error.into());
+ }
+ };
+ run.engine
+ .runtime
+ .block_on(call.respond(std::future::ready(response)))
+ }
+ Call::Decision(call) => {
+ let response = match reply {
+ CallReply::Error { error } => Ok(Err(LibsyError::external(
+ "host decision call",
+ std::io::Error::other(error),
+ ))),
+ CallReply::Response { response, .. } => {
+ serde_json::from_value(response).map(Ok)
+ }
+ };
+ let response = match response {
+ Ok(response) => response,
+ Err(error) => {
+ lifecycle.pending.insert(call_id, Call::Decision(call));
+ return Err(error.into());
+ }
+ };
+ run.engine
+ .runtime
+ .block_on(call.respond(std::future::ready(response)))
+ }
+ };
+ if result.is_err() {
+ lifecycle.status = RunStatus::Failed;
+ lifecycle.pending.clear();
+ }
+ drop(lifecycle);
+ if result.is_err() {
+ run.state.stop(RunStatus::Failed);
+ }
+ Ok(result?)
+ })
+}
+
+/// Signals cancellation without waiting for callbacks.
+/// # Safety
+/// The run must remain live for the duration of this call.
+#[unsafe(no_mangle)]
+pub unsafe extern "C" fn sy_run_cancel(run: *const SyRun) {
+ let _ = catch_unwind(AssertUnwindSafe(|| {
+ if let Some(run) = unsafe { run.as_ref() } {
+ run.state.stop(RunStatus::Cancelled);
+ }
+ }));
+}
+
+/// Cancels a run, waits for accepted callbacks to return, and frees it.
+/// # Safety
+/// Pass a handle from `sy_run_new` once, or NULL. Other calls using it must have returned.
+/// Accepted callbacks may still be active; keep their state live until they return.
+/// Do not call from a completion callback or Tokio task.
+#[unsafe(no_mangle)]
+pub unsafe extern "C" fn sy_run_free(run: *mut SyRun) {
+ if !run.is_null() {
+ let _ = catch_unwind(AssertUnwindSafe(|| drop(unsafe { Box::from_raw(run) })));
+ }
+}
+
+/// Frees the router handle; existing runs retain its engine.
+/// # Safety
+/// Pass a handle from `sy_router_new` once, with no concurrent calls using it, or NULL.
+#[unsafe(no_mangle)]
+pub unsafe extern "C" fn sy_router_free(router: *mut SyRouter) {
+ if !router.is_null() {
+ let _ = catch_unwind(AssertUnwindSafe(|| drop(unsafe { Box::from_raw(router) })));
+ }
+}
+
/// Frees a library-owned error or JSON string.
/// # Safety
/// Pass a returned string once, with no remaining readers, or NULL.
diff --git a/crates/switchyard-ffi/src/tests.rs b/crates/switchyard-ffi/src/tests.rs
index 29ec3f397..2cc53ceeb 100644
--- a/crates/switchyard-ffi/src/tests.rs
+++ b/crates/switchyard-ffi/src/tests.rs
@@ -5,6 +5,46 @@
use super::*;
use serde_json::{Value, json};
+use std::sync::atomic::{AtomicUsize, Ordering};
+use std::sync::mpsc;
+use std::time::Duration;
+
+type Completion = std::result::Result<CString, CString>;
+
+unsafe extern "C" fn complete(user_data: usize, error: *mut c_char, output: *mut c_char) {
+ let sender = unsafe { &*(user_data as *const mpsc::Sender<Completion>) };
+ let output = (!output.is_null()).then(|| unsafe { CString::from_raw(output) });
+ let result = if error.is_null() {
+ Ok(output.unwrap_or_default())
+ } else {
+ Err(unsafe { CString::from_raw(error) })
+ };
+ let _ = sender.send(result);
+}
+
+unsafe extern "C" fn complete_once(user_data: usize, error: *mut c_char, output: *mut c_char) {
+ let sender = unsafe { Box::from_raw(user_data as *mut mpsc::Sender<Completion>) };
+ unsafe { complete(&*sender as *const _ as usize, error, output) };
+}
+
+fn next_async(run: &SyRun, sender: &mpsc::Sender<Completion>) -> std::result::Result<(), String> {
+ let sender = Box::into_raw(Box::new(sender.clone()));
+ let error = unsafe { sy_run_next_async(run, Some(complete_once), sender as usize) };
+ if error.is_null() {
+ Ok(())
+ } else {
+ drop(unsafe { Box::from_raw(sender) });
+ Err(error_message(error))
+ }
+}
+
+fn received_step(receiver: &mpsc::Receiver<Completion>) -> Value {
+ let output = receiver
+ .recv_timeout(Duration::from_secs(5))
+ .unwrap()
+ .unwrap();
+ serde_json::from_slice(output.as_bytes()).unwrap()
+}
// ABI generations are independent of package releases.
#[test]
@@ -36,6 +76,37 @@ fn check(error: *mut c_char) {
}
}
+fn error_message(error: *mut c_char) -> String {
+ assert!(!error.is_null(), "expected an error");
+ unsafe { CString::from_raw(error) }.into_string().unwrap()
+}
+
+fn router(algorithm: Value) -> Box<SyRouter> {
+ let config = json!({
+ "algorithm": algorithm,
+ "targets": {"capable": "capable", "efficient": "efficient", "judge": "judge"},
+ });
+ let config = CString::new(config.to_string()).unwrap();
+ let mut router = std::ptr::null_mut();
+ check(unsafe { sy_router_new(config.as_ptr(), &mut router) });
+ assert!(!router.is_null());
+ unsafe { Box::from_raw(router) }
+}
+
+fn run(router: &SyRouter, input: Value) -> Box<SyRun> {
+ let input = CString::new(input.to_string()).unwrap();
+ let mut run = std::ptr::null_mut();
+ check(unsafe { sy_run_new(router, input.as_ptr(), &mut run) });
+ assert!(!run.is_null());
+ unsafe { Box::from_raw(run) }
+}
+
+fn next(run: &SyRun) -> Value {
+ let (sender, receiver) = mpsc::channel();
+ next_async(run, &sender).unwrap();
+ received_step(&receiver)
+}
+
fn translated(
call: unsafe extern "C" fn(*const c_char, *mut *mut c_char) -> *mut c_char,
input: Value,
@@ -49,6 +120,224 @@ fn translated(
result.unwrap()
}
+fn advisor_router() -> Box<SyRouter> {
+ router(json!({
+ "type": "advisor", "executor_target": "capable", "advisor_target": "judge",
+ "gate_trigger": "pattern", "gate_trigger_pattern": "NEVER",
+ }))
+}
+
+fn advisor_input() -> Value {
+ json!({"request": {}, "models": {"efficient": ["capable"], "judge": ["judge"]}})
+}
+
+fn pending_advisor() -> (Box<SyRun>, u64) {
+ let router = advisor_router();
+ let run = run(&router, advisor_input());
+ let call = next(&run);
+ assert_eq!(call["type"], "call_model");
+ (run, call["id"].as_u64().unwrap())
+}
+
+#[test]
+fn completed_runs_reject_further_reads() {
+ let router = router(json!({"type": "passthrough", "target": "capable"}));
+ let run = run(
+ &router,
+ json!({"request": {}, "models": {"any": ["capable"]}}),
+ );
+ unsafe { sy_router_free(Box::into_raw(router)) };
+ assert_eq!(next(&run)["type"], "done");
+ let (sender, receiver) = mpsc::channel();
+ for _ in 0..2 {
+ assert_eq!(
+ next_async(&run, &sender).unwrap_err(),
+ "routing already completed"
+ );
+ assert!(matches!(
+ receiver.try_recv(),
+ Err(mpsc::TryRecvError::Empty)
+ ));
+ }
+ unsafe { sy_run_free(Box::into_raw(run)) };
+}
+
+// Streamed answers still use the buffered Done contract without losing the answer.
+#[test]
+fn noop_outcomes_return_buffered_answers() {
+ let router = router(json!({"type": "noop"}));
+ for stream in [false, true] {
+ let run = run(
+ &router,
+ json!({
+ "request": {"model": "capable", "stream": stream},
+ "models": {"any": ["capable"]},
+ }),
+ );
+ let done = next(&run);
+ assert_eq!(done["type"], "done");
+ assert_eq!(done["selected_model_ids"], json!(["capable"]));
+ assert_eq!(done["request"]["model"], "capable");
+ assert_eq!(done["request"]["stream"], stream);
+ assert_eq!(done["response"]["id"], "switchyard-noop");
+ assert_eq!(done["response"]["model"], "capable");
+ assert_eq!(done["response"]["outputs"][0]["content"][0]["text"], "OK");
+ }
+}
+
+// Cancellation must interrupt response aggregation, not just waiting for a routing step.
+#[test]
+fn streamed_outcomes_propagate_errors_and_cancellation() {
+ let router = router(json!({"type": "noop"}));
+ for (cancel, expected_error, terminal_error) in [
+ (false, "stream failed", "routing failed"),
+ (true, "routing cancelled", "routing cancelled"),
+ ] {
+ let (polled, started) = mpsc::channel();
+ let response = Response {
+ llm_response: LlmResponse::Stream(Box::pin(futures::stream::poll_fn(move |_| {
+ let _ = polled.send(());
+ if cancel {
+ std::task::Poll::Pending
+ } else {
+ std::task::Poll::Ready(Some(Err(switchyard_protocol::LlmClientError::General(
+ "stream failed".into(),
+ ))))
+ }
+ }))),
+ metadata: None,
+ upstream_headers: Default::default(),
+ };
+ let outcome = switchyard_libsy::RoutingOutcome::answered(
+ "capable".into(),
+ Request::default(),
+ response,
+ );
+ let run = Box::new(SyRun::new(
+ Box::pin(futures::stream::iter([Ok(Step::Done(Box::new(outcome)))])),
+ Arc::clone(&router.engine),
+ ));
+ let (sender, receiver) = mpsc::channel();
+ next_async(&run, &sender).unwrap();
+ started.recv_timeout(Duration::from_secs(5)).unwrap();
+ if cancel {
+ unsafe { sy_run_cancel(&*run) };
+ }
+ let error = receiver
+ .recv_timeout(Duration::from_secs(5))
+ .unwrap()
+ .unwrap_err();
+ assert_eq!(error.to_str().unwrap(), expected_error);
+ assert_eq!(next_async(&run, &sender).unwrap_err(), terminal_error);
+ unsafe { sy_run_free(Box::into_raw(run)) };
+ }
+}
+
+// Fallback candidates retain their request when the selected model's request was rewritten.
+#[test]
+fn completed_runs_preserve_fallback_requests() {
+ let router = router(json!({"type": "passthrough", "target": "capable"}));
+ for fallback in [
+ Some(switchyard_protocol::text_request(None, "original")),
+ None,
+ ] {
+ let expected = serde_json::to_value(&fallback).unwrap();
+ let mut outcome = switchyard_libsy::RoutingOutcome::route_to(
+ "capable".into(),
+ vec!["efficient".into()],
+ Request {
+ llm_request: switchyard_protocol::text_request(None, "rewritten"),
+ ..Default::default()
+ },
+ );
+ outcome.fallback_request = fallback.map(|llm_request| Request {
+ llm_request,
+ ..Default::default()
+ });
+ let run = SyRun::new(
+ Box::pin(futures::stream::iter([Ok(Step::Done(Box::new(outcome)))])),
+ Arc::clone(&router.engine),
+ );
+ let done = next(&run);
+ assert_eq!(done["selected_model_ids"], json!(["capable", "efficient"]));
+ assert_eq!(
+ done["request"]["messages"][0]["content"][0]["text"],
+ "rewritten"
+ );
+ assert_eq!(done.get("fallback_request"), Some(&expected));
+ }
+}
+
+#[test]
+fn cancellation_completes_pending_read_and_rejects_replies() {
+ let (run, id) = pending_advisor();
+ let (sender, receiver) = mpsc::channel();
+ next_async(&run, &sender).unwrap();
+ assert_eq!(
+ next_async(&run, &sender).unwrap_err(),
+ "routing read already in progress"
+ );
+ unsafe { sy_run_cancel(&*run) };
+ let error = receiver
+ .recv_timeout(Duration::from_secs(5))
+ .unwrap()
+ .unwrap_err();
+ assert_eq!(error.to_str().unwrap(), "routing cancelled");
+ let error = unsafe { sy_run_respond(&*run, id, c"{\"response\":{}}".as_ptr()) };
+ assert_eq!(error_message(error), "routing cancelled");
+}
+
+// A malformed reply must not consume its call ID.
+#[test]
+fn invalid_model_replies_preserve_pending_calls() {
+ let (run, id) = pending_advisor();
+ let error = unsafe {
+ sy_run_respond(
+ &*run,
+ id,
+ c"{\"response\":{\"outputs\":\"wrong\"}}".as_ptr(),
+ )
+ };
+ assert!(!error_message(error).is_empty());
+ check(unsafe { sy_run_respond(&*run, id, c"{\"response\":{}}".as_ptr()) });
+ let error = unsafe { sy_run_respond(&*run, id, c"{\"response\":{}}".as_ptr()) };
+ assert_eq!(error_message(error), "unknown call ID");
+ assert_eq!(next(&run)["type"], "done");
+}
+
+// Decision replies have a different schema, but the same retryable validation boundary.
+#[test]
+fn invalid_decision_replies_preserve_pending_calls() {
+ let router = router(json!({
+ "type": "llm_classifier", "classifier_target": "judge",
+ "strong_target": "capable", "weak_target": "efficient", "fail_open": false,
+ "decision": {"cutoff": 0.5, "candidates": {"a": "capable", "b": "efficient"}, "evidence": {}},
+ }));
+ let run = run(
+ &router,
+ json!({
+ "request": {},
+ "models": {
+ "any": ["capable", "efficient"], "capable": ["capable"],
+ "efficient": ["efficient"], "judge": ["judge"],
+ },
+ }),
+ );
+ let call = next(&run);
+ assert_eq!(call["type"], "call_decision");
+ assert_eq!(call["model"], "judge");
+ let id = call["id"].as_u64().unwrap();
+ let error = unsafe { sy_run_respond(&*run, id, c"{\"response\":{}}".as_ptr()) };
+ assert!(!error_message(error).is_empty());
+ let reply = CString::new(json!({"response": {"answers": {"route": {"value": {
+ "type": "choice", "data": {"selected": "advantage", "probabilities": {"advantage": 0.8, "no_advantage": 0.2}},
+ }}}}}).to_string()).unwrap();
+ check(unsafe { sy_run_respond(&*run, id, reply.as_ptr()) });
+ let done = next(&run);
+ assert_eq!(done["type"], "done");
+ assert_eq!(done["selected_model_ids"], json!(["capable", "efficient"]));
+}
+
// Translation exports return owned JSON strings that the caller releases through the ABI.
#[test]
fn translation_exports_return_owned_json() {
@@ -76,3 +365,264 @@ fn translation_exports_return_owned_json() {
);
assert_eq!(response["outputs"][0]["content"][0]["text"], "hello");
}
+
+// Pending reads must yield the runtime worker, not consume blocking threads.
+#[test]
+fn pending_reads_yield_runtime_worker() {
+ let thread_starts = Arc::new(AtomicUsize::new(0));
+ let observed = Arc::clone(&thread_starts);
+ let mut router = advisor_router();
+ Arc::get_mut(&mut router.engine).unwrap().runtime = tokio::runtime::Builder::new_multi_thread()
+ .worker_threads(1)
+ .enable_all()
+ .on_thread_start(move || {
+ observed.fetch_add(1, Ordering::SeqCst);
+ })
+ .build()
+ .unwrap();
+ let (sender, receiver) = mpsc::channel();
+ let mut runs = Vec::new();
+ for _ in 0..32 {
+ let run = run(&router, advisor_input());
+ next_async(&run, &sender).unwrap();
+ assert_eq!(received_step(&receiver)["type"], "call_model");
+ next_async(&run, &sender).unwrap();
+ runs.push(run);
+ }
+ let (scheduled, sentinel) = mpsc::channel();
+ router.engine.runtime.spawn(async move {
+ let _ = scheduled.send(());
+ });
+ sentinel.recv_timeout(Duration::from_secs(5)).unwrap();
+ assert_eq!(thread_starts.load(Ordering::SeqCst), 1);
+ drop(runs);
+ for _ in 0..32 {
+ let error = receiver
+ .recv_timeout(Duration::from_secs(5))
+ .unwrap()
+ .unwrap_err();
+ assert_eq!(error.to_str().unwrap(), "routing cancelled");
+ }
+}
+
+struct BlockedCompletion {
+ sender: mpsc::Sender<Completion>,
+ resume: Mutex<mpsc::Receiver<()>>,
+ finished: mpsc::Sender<()>,
+}
+
+unsafe extern "C" fn complete_after_release(
+ user_data: usize,
+ error: *mut c_char,
+ output: *mut c_char,
+) {
+ let state = unsafe { &*(user_data as *const BlockedCompletion) };
+ unsafe { complete(&state.sender as *const _ as usize, error, output) };
+ let _ = state.resume.lock().recv();
+ let _ = state.finished.send(());
+}
+
+// Keep another runtime owner so runtime shutdown cannot mask an early free.
+#[test]
+fn free_waits_for_cancelled_callback_to_return() {
+ let engine = Arc::new(Engine {
+ algorithm: Arc::new(switchyard_libsy::Noop {}),
+ runtime: tokio::runtime::Builder::new_multi_thread()
+ .worker_threads(2)
+ .enable_all()
+ .build()
+ .unwrap(),
+ });
+ let (sender, receiver) = mpsc::channel();
+ let (resume, paused) = mpsc::channel();
+ let (finished, returned) = mpsc::channel();
+ let state = Box::into_raw(Box::new(BlockedCompletion {
+ sender,
+ resume: Mutex::new(paused),
+ finished,
+ }));
+ let run = Box::new(SyRun::new(
+ Box::pin(futures::stream::pending()),
+ Arc::clone(&engine),
+ ));
+ check(unsafe { sy_run_next_async(&*run, Some(complete_after_release), state as usize) });
+ let run = Box::into_raw(run) as usize;
+ let (freed, closed) = mpsc::channel();
+ let freeing = std::thread::spawn(move || {
+ unsafe { sy_run_free(run as *mut SyRun) };
+ let _ = freed.send(());
+ });
+ let cancellation = receiver.recv_timeout(Duration::from_secs(5));
+ let premature_free = closed.recv_timeout(Duration::from_millis(50));
+ let _ = resume.send(());
+ returned.recv_timeout(Duration::from_secs(5)).unwrap();
+ if premature_free.is_err() {
+ closed.recv_timeout(Duration::from_secs(5)).unwrap();
+ }
+ freeing.join().unwrap();
+ drop(unsafe { Box::from_raw(state) });
+ assert!(matches!(
+ premature_free,
+ Err(mpsc::RecvTimeoutError::Timeout)
+ ));
+ assert_eq!(
+ cancellation.unwrap().unwrap_err().to_str().unwrap(),
+ "routing cancelled"
+ );
+}
+
+// Runs may share call IDs without sharing replies or cancellation.
+#[test]
+fn async_runs_complete_independently() {
+ let router = advisor_router();
+ let (sender, receiver) = mpsc::channel();
+ let runs: Vec<_> = (0..3).map(|_| run(&router, advisor_input())).collect();
+ let mut ids = Vec::new();
+ for run in &runs {
+ next_async(run, &sender).unwrap();
+ ids.push(received_step(&receiver)["id"].as_u64().unwrap());
+ next_async(run, &sender).unwrap();
+ }
+ for (index, text) in [(1, "second"), (0, "first")] {
+ if index == 0 {
+ unsafe { sy_run_cancel(&*runs[2]) };
+ let error = receiver
+ .recv_timeout(Duration::from_secs(5))
+ .unwrap()
+ .unwrap_err();
+ assert_eq!(error.to_str().unwrap(), "routing cancelled");
+ }
+ let reply = CString::new(
+ json!({
+ "response": switchyard_protocol::text_response(None, text),
+ })
+ .to_string(),
+ )
+ .unwrap();
+ check(unsafe { sy_run_respond(&*runs[index], ids[index], reply.as_ptr()) });
+ let done = received_step(&receiver);
+ assert_eq!(done["type"], "done");
+ assert_eq!(done["response"]["outputs"][0]["content"][0]["text"], text);
+ }
+}
+
+// Host fallback attribution uses target IDs, not provider model names.
+#[test]
+fn advisor_attributes_host_fallback() {
+ let router = advisor_router();
+ for served_model in [None, Some("fallback")] {
+ let input = json!({
+ "request": {}, "models": {"efficient": ["capable", "fallback"], "judge": ["judge"]},
+ });
+ let run = run(&router, input);
+ let call = next(&run);
+ assert_eq!(call["models"], json!(["capable", "fallback"]));
+ let mut reply = json!({
+ "response": switchyard_protocol::text_response(Some("provider-model".into()), "answer"),
+ });
+ if let Some(model) = served_model {
+ reply["served_model"] = json!(model);
+ }
+ let reply = CString::new(reply.to_string()).unwrap();
+ check(unsafe { sy_run_respond(&*run, call["id"].as_u64().unwrap(), reply.as_ptr()) });
+ let done = next(&run);
+ let expected = served_model.unwrap_or("capable");
+ assert_eq!(done["type"], "done");
+ assert_eq!(done["selected_model_ids"], json!([expected]));
+ assert_eq!(done["request"]["model"], expected);
+ assert_eq!(done["response"]["model"], "provider-model");
+ }
+}
+
+// Child sessions keep separate phases and clear them after the final request.
+#[test]
+fn plan_execute_isolates_and_finishes_child_sessions() {
+ let router = router(json!({
+ "type": "plan_execute", "capable_target": "capable", "efficient_target": "efficient",
+ }));
+ for (agent, is_subagent, is_final, edited, expected) in [
+ ("a", true, false, true, "efficient"),
+ ("b", true, false, false, "capable"),
+ ("root", false, false, false, "capable"),
+ ("a", true, true, false, "efficient"),
+ ("a", true, false, false, "capable"),
+ ] {
+ let request = if edited {
+ json!({"messages": [{"role": "assistant", "content": [{
+ "type": "tool_call", "id": "edit", "name": "apply_patch", "arguments": {},
+ }]}]})
+ } else {
+ json!({})
+ };
+ let input = json!({
+ "request": request, "models": {"capable": ["capable"], "efficient": ["efficient"]},
+ "metadata": {"session_id": "s", "agent_id": agent, "is_subagent": is_subagent,
+ "session_final": is_final},
+ });
+ let run = run(&router, input);
+ assert_eq!(
+ next(&run)["selected_model_ids"],
+ json!([expected]),
+ "{agent}"
+ );
+ }
+}
+
+// Child identity alone must not activate the delegated-work policy.
+#[test]
+fn delegated_work_uses_subagent_model_groups() {
+ let router = router(json!({
+ "type": "passthrough", "target": "capable",
+ "subagents": {"type": "passthrough", "target": "efficient"},
+ }));
+ for (metadata, expected) in [
+ (Value::Null, "capable"),
+ (json!({"is_subagent": true}), "capable"),
+ (
+ json!({"is_subagent": true, "is_delegated_work": true}),
+ "efficient",
+ ),
+ ] {
+ let input = json!({
+ "request": {}, "models": {"any": ["capable"]},
+ "subagent_models": {"any": ["efficient"]}, "metadata": metadata,
+ });
+ let run = run(&router, input);
+ assert_eq!(next(&run)["selected_model_ids"], json!([expected]));
+ }
+}
+
+#[test]
+fn invalid_router_input_returns_owned_errors() {
+ for input in [std::ptr::null(), c"{".as_ptr()] {
+ let mut router = std::ptr::dangling_mut::<SyRouter>();
+ let error = unsafe { sy_router_new(input, &mut router) };
+ assert!(!error.is_null());
+ assert!(router.is_null());
+ let message = unsafe { CStr::from_ptr(error) }.to_str().unwrap();
+ assert!(!message.is_empty());
+ unsafe { sy_string_free(error) };
+ }
+ let error = unsafe { sy_router_new(c"{".as_ptr(), std::ptr::null_mut()) };
+ assert!(!error.is_null());
+ unsafe { sy_string_free(error) };
+}
+
+// Unwinding panics become owned errors at the C boundary.
+#[test]
+fn panics_return_owned_errors() {
+ let run = SyRun::new(
+ Box::pin(futures::stream::poll_fn(|_| panic!("test panic"))),
+ Arc::new(Engine {
+ algorithm: Arc::new(switchyard_libsy::Noop {}),
+ runtime: Runtime::new().unwrap(),
+ }),
+ );
+ let (sender, receiver) = mpsc::channel();
+ next_async(&run, &sender).unwrap();
+ let error = receiver
+ .recv_timeout(Duration::from_secs(5))
+ .unwrap()
+ .unwrap_err();
+ assert_eq!(error.to_str().unwrap(), "Switchyard binding panicked");
+}