Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.
Rust
feat(ffi): add asynchronous routing handles (#1077)
Alex Fournierb0fa4fb201e5
diff --git a/Cargo.lock b/Cargo.lockindex e7f816931..a9ae848f0 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -5415,10 +5415,16 @@ dependencies = [name = "switchyard-ffi"version = "0.3.0"dependencies = [+ "futures",+ "parking_lot","serde","serde_json",+ "switchyard-libsy","switchyard-protocol",+ "switchyard-runner","switchyard-translation",+ "tokio",+ "tokio-util",][[package]]diff --git a/crates/switchyard-ffi/Cargo.toml b/crates/switchyard-ffi/Cargo.tomlindex c113438ed..96e48d3b9 100644--- a/crates/switchyard-ffi/Cargo.toml+++ b/crates/switchyard-ffi/Cargo.toml@@ -13,7 +13,13 @@ publish = falsecrate-type = ["cdylib", "staticlib"][dependencies]+futures.workspace = true+parking_lot.workspace = trueserde.workspace = trueserde_json.workspace = true+switchyard-libsy.workspace = trueswitchyard-protocol.workspace = true+switchyard-runner.workspace = trueswitchyard-translation.workspace = true+tokio.workspace = true+tokio-util = { version = "0.7", features = ["rt"] }diff --git a/crates/switchyard-ffi/README.md b/crates/switchyard-ffi/README.mdindex 4b1ec9866..9a20b330d 100644--- a/crates/switchyard-ffi/README.md+++ b/crates/switchyard-ffi/README.md@@ -1,21 +1,43 @@-# C translation interface+# C routing interface-An experimental C interface to Switchyard's request and response codecs. The-interface uses NUL-terminated JSON strings. Ownership rules are in+An experimental C interface to libsy. The host supplies eligible models and+executes model and decision calls. The interface uses opaque handles and+NUL-terminated JSON strings. Ownership and concurrency rules are in[`include/switchyard.h`](include/switchyard.h).## JSON payloads| Function | Input || --- | --- |+| `sy_router_new` | `{algorithm, targets}`: an `AlgorithmSpec` and target-alias-to-model-ID map |+| `sy_run_new` | `{request, models, subagent_models?, metadata?}` |+| `sy_run_respond` | `{response, served_model?}` or `{error}` || `sy_decode_request`, `sy_decode_response` | `{format, body}` || `sy_encode_request` | `{format, request, model}` |`request` and model responses use Switchyard's normalized protocol JSON.-`sy_encode_request` applies the selected model before encoding the request.+`models` and `subagent_models` map category names to ordered model-ID lists.+Optional metadata fields are `session_id`, `agent_id`, `is_subagent`,+`is_delegated_work`, and `session_final`.++For model replies, `served_model` identifies the target that answered, which+can differ from both the first candidate and the provider's response model name.+Decision replies contain a normalized `DecisionResponse`; `served_model` does+not apply to them. Error replies contain a string.++Each accepted read delivers one step or an error through its callback:++- `call_model`: `{type, id, models, request}`+- `call_decision`: `{type, id, model, request}`+- `done`: `{type, selected_model_ids, request, fallback_request, response, metadata}`++A callback error or `done` ends the run. Outcomes retain ordered targets, the+prepared request, an optional buffered answer, and optional routing evidence.+The host executes the answer call when `response` is null.+Streamed routing answers are collected into a buffered response before `done`.+Cancellation interrupts collection; stream errors end the run with a callback error.+When `fallback_request` is non-null, use it for candidates after the first.+Content-policy denials must not advance to those fallback candidates.Wire formats are `openai_chat`, `openai_responses`, and `anthropic_messages`.The translation functions use the existing Switchyard codecs.--Fallible calls return NULL on success or an owned error string. Free every-returned error or JSON string with `sy_string_free`.diff --git a/crates/switchyard-ffi/include/switchyard.h b/crates/switchyard-ffi/include/switchyard.hindex ded662702..3ff8fe08a 100644--- a/crates/switchyard-ffi/include/switchyard.h+++ b/crates/switchyard-ffi/include/switchyard.h@@ -10,13 +10,34 @@#define SY_ABI_VERSION 1uint32_t sy_abi_version(void);+typedef struct SyRouter SyRouter;+typedef struct SyRun SyRun;+typedef void (*SyNextCallback)(uintptr_t user_data, char *error, char *step_json);+// Fallible calls return NULL on success or an owned error string.// JSON inputs are borrowed for the call and must be NUL-terminated.-// Free all output strings with sy_string_free.+// Free all output strings, including callback strings, with sy_string_free.+char *sy_router_new(const char *config_json, SyRouter **out);+char *sy_run_new(const SyRouter *router, const char *input_json, SyRun **out);++// One pending read per run; other runs execute independently. Replies and cancellation may overlap.+// NULL accepts one read (one callback, even if cancelled); errors reject it (no callback).+// Callbacks may overlap or run before return. Keep callback state alive until they return.+// Callbacks must return promptly, not unwind, and not call sy_run_respond or free handles.+char *sy_run_next_async(const SyRun *run, SyNextCallback callback, uintptr_t user_data);+char *sy_run_respond(const SyRun *run, uint64_t call_id, const char *reply_json);+char *sy_decode_request(const char *input_json, char **out_json);char *sy_encode_request(const char *input_json, char **out_json);char *sy_decode_response(const char *input_json, char **out_json);+// Signals cancellation without waiting for callbacks.+void sy_run_cancel(const SyRun *run);+// Free a run once, after other calls using it return; accepted callbacks may still be active.+// Cancels and waits for those callbacks. Do not call from a callback or Tokio task.+void sy_run_free(SyRun *run);+// Free a router once, with no calls using it. Existing runs retain their engine.+void sy_router_free(SyRouter *router);void sy_string_free(char *string);#endifdiff --git a/crates/switchyard-ffi/src/lib.rs b/crates/switchyard-ffi/src/lib.rsindex 2a0c73a09..b004b5ecd 100644--- a/crates/switchyard-ffi/src/lib.rs+++ b/crates/switchyard-ffi/src/lib.rs@@ -1,15 +1,28 @@// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.// SPDX-License-Identifier: Apache-2.0-//! C bridge to Switchyard's request and response codecs.+//! Pull-based C bridge to libsy.+use std::collections::{BTreeMap, HashMap};use std::ffi::{CStr, CString, c_char};use std::panic::{AssertUnwindSafe, catch_unwind};+use std::sync::Arc;+use futures::{FutureExt, StreamExt};+use parking_lot::Mutex;use serde::{Deserialize, Serialize};-use switchyard_protocol::{LlmRequest, ModelId, WireFormat};+use switchyard_libsy::{Algorithm, Call, LibsyError, RuntimeModels, State, Step, StepStream};+use switchyard_protocol::{+ AggLlmResponse, Category, DecisionRequest, LlmRequest, LlmResponse, Metadata, ModelId, Request,+ Response, WireFormat,+};+use switchyard_runner::AlgorithmSpec;+use tokio::runtime::Runtime;+use tokio_util::sync::CancellationToken;+use tokio_util::task::TaskTracker;type Result<T> = std::result::Result<T, Box<dyn std::error::Error>>;+type ModelGroups = HashMap<String, Option<Vec<ModelId>>>;/// Returns the compatibility generation for C signatures and JSON payloads.#[unsafe(no_mangle)]@@ -17,6 +30,31 @@ pub extern "C" fn sy_abi_version() -> u32 {1}+#[derive(Deserialize)]+struct RouterConfig {+ algorithm: AlgorithmSpec,+ targets: BTreeMap<String, ModelId>,+}++#[derive(Deserialize)]+struct RunInput {+ request: LlmRequest,+ models: ModelGroups,+ subagent_models: Option<ModelGroups>,+ metadata: Option<RequestMetadata>,+}++#[derive(Deserialize)]+struct RequestMetadata {+ session_id: Option<String>,+ agent_id: Option<String>,+ #[serde(default)]+ is_subagent: bool,+ #[serde(default)]+ is_delegated_work: bool,+ session_final: Option<bool>,+}+#[derive(Deserialize)]struct DecodeInput {format: WireFormat,@@ -30,6 +68,254 @@ struct EncodeInput {model: ModelId,}+#[derive(Deserialize)]+#[serde(untagged)]+enum CallReply {+ Error {+ error: String,+ },+ Response {+ response: serde_json::Value,+ served_model: Option<ModelId>,+ },+}++#[derive(Serialize)]+struct OutcomeMetadataRef<'a> {+ outcome_id: &'a str,+ algorithm: &'a str,+ evidence: Option<&'a serde_json::Value>,+}++#[derive(Serialize)]+#[serde(tag = "type", rename_all = "snake_case")]+enum RunStep<'a> {+ CallModel {+ id: u64,+ models: &'a [ModelId],+ request: &'a LlmRequest,+ },+ CallDecision {+ id: u64,+ model: &'a ModelId,+ request: &'a DecisionRequest,+ },+ Done {+ selected_model_ids: &'a [ModelId],+ request: &'a LlmRequest,+ fallback_request: Option<&'a LlmRequest>,+ response: Option<&'a AggLlmResponse>,+ metadata: Option<OutcomeMetadataRef<'a>>,+ },+}++struct Engine {+ algorithm: Arc<dyn Algorithm>,+ runtime: Runtime,+}++/// Algorithm handle; routing runs retain its engine.+pub struct SyRouter {+ engine: Arc<Engine>,+}++struct Reader {+ stream: Option<StepStream>,+ next_id: u64,+}++enum RunStatus {+ Active,+ Completed,+ Cancelled,+ Failed,+}++impl RunStatus {+ fn require_active(&self) -> Result<()> {+ match self {+ Self::Active => Ok(()),+ Self::Completed => Err("routing already completed".into()),+ Self::Cancelled => Err("routing cancelled".into()),+ Self::Failed => Err("routing failed".into()),+ }+ }+}++struct Lifecycle {+ status: RunStatus,+ is_reading: bool,+ pending: HashMap<u64, Call>,+}++struct RunState {+ reader: tokio::sync::Mutex<Reader>,+ lifecycle: Mutex<Lifecycle>,+ cancellation: CancellationToken,+}++/// Routing run that owns pending calls and callback cleanup.+pub struct SyRun {+ state: Arc<RunState>,+ tasks: TaskTracker,+ // Tasks retain only run state; free joins them before dropping the runtime.+ engine: Arc<Engine>,+}++impl SyRun {+ fn new(stream: StepStream, engine: Arc<Engine>) -> Self {+ Self {+ state: Arc::new(RunState {+ reader: tokio::sync::Mutex::new(Reader {+ stream: Some(stream),+ next_id: 1,+ }),+ lifecycle: Mutex::new(Lifecycle {+ status: RunStatus::Active,+ is_reading: false,+ pending: HashMap::new(),+ }),+ cancellation: CancellationToken::new(),+ }),+ tasks: TaskTracker::new(),+ engine,+ }+ }+}++impl Drop for SyRun {+ fn drop(&mut self) {+ self.state.stop(RunStatus::Cancelled);+ self.tasks.close();+ self.engine.runtime.block_on(self.tasks.wait());+ }+}++struct ReadGuard {+ state: Arc<RunState>,+}++impl Drop for ReadGuard {+ fn drop(&mut self) {+ self.state.lifecycle.lock().is_reading = false;+ }+}++impl RunState {+ fn begin_read(self: &Arc<Self>) -> Result<ReadGuard> {+ let mut lifecycle = self.lifecycle.lock();+ lifecycle.status.require_active()?;+ if lifecycle.is_reading {+ return Err("routing read already in progress".into());+ }+ lifecycle.is_reading = true;+ Ok(ReadGuard {+ state: Arc::clone(self),+ })+ }++ // Serialize cancellation with reply delivery.+ fn stop(&self, status: RunStatus) {+ {+ let mut lifecycle = self.lifecycle.lock();+ if matches!(lifecycle.status, RunStatus::Active) {+ lifecycle.status = status;+ }+ lifecycle.pending.clear();+ }+ self.cancellation.cancel();+ if let Ok(mut reader) = self.reader.try_lock() {+ reader.stream.take();+ }+ }++ async fn next_step(&self, reader: &mut Reader) -> Result<CString> {+ self.lifecycle.lock().status.require_active()?;+ let stream = reader.stream.as_mut().ok_or("routing stream is closed")?;+ let step = tokio::select! {+ biased;+ _ = self.cancellation.cancelled() => None,+ step = stream.next() => step,+ };+ self.lifecycle.lock().status.require_active()?;+ let step = step.ok_or("routing stream ended without an outcome")??;+ let id = reader.next_id;+ reader.next_id += 1;+ match step {+ Step::CallModel(call) => {+ let output = encode_json(&RunStep::CallModel {+ id,+ models: &call.models,+ request: &call.request.llm_request,+ })?;+ let mut lifecycle = self.lifecycle.lock();+ lifecycle.status.require_active()?;+ lifecycle.pending.insert(id, Call::Model(call));+ Ok(output)+ }+ Step::CallDecision(call) => {+ let output = encode_json(&RunStep::CallDecision {+ id,+ model: &call.model,+ request: &call.request,+ })?;+ let mut lifecycle = self.lifecycle.lock();+ lifecycle.status.require_active()?;+ lifecycle.pending.insert(id, Call::Decision(call));+ Ok(output)+ }+ Step::Done(outcome) => {+ let response = match outcome.response {+ None => None,+ Some(response) => tokio::select! {+ biased;+ _ = self.cancellation.cancelled() => return Err("routing cancelled".into()),+ response = response.llm_response.into_agg() => Some(response?),+ },+ };+ let output = encode_json(&RunStep::Done {+ selected_model_ids: &outcome.selected_model_ids,+ request: &outcome.request.llm_request,+ fallback_request: outcome+ .fallback_request+ .as_ref()+ .map(|request| &request.llm_request),+ response: response.as_ref(),+ metadata: outcome+ .metadata+ .as_ref()+ .map(|metadata| OutcomeMetadataRef {+ outcome_id: metadata.outcome_id(),+ algorithm: &metadata.algorithm,+ evidence: metadata.evidence.as_ref(),+ }),+ })?;+ let mut lifecycle = self.lifecycle.lock();+ lifecycle.status.require_active()?;+ lifecycle.status = RunStatus::Completed;+ lifecycle.pending.clear();+ reader.stream.take();+ Ok(output)+ }+ }+ }+}++impl ReadGuard {+ async fn next(self) -> Result<CString> {+ let mut reader = self.state.reader.lock().await;+ let result = AssertUnwindSafe(self.state.next_step(&mut reader))+ .catch_unwind()+ .await+ .unwrap_or_else(|_| Err("Switchyard binding panicked".into()));+ if result.is_err() {+ self.state.stop(RunStatus::Failed);+ reader.stream.take();+ }+ result+ }+}+fn error_string(message: String) -> *mut c_char {match CString::new(message) {Ok(message) => message.into_raw(),@@ -125,6 +411,225 @@ pub unsafe extern "C" fn sy_decode_response(})}+fn parse_model_groups(groups: ModelGroups) -> Result<HashMap<Category, Vec<ModelId>>> {+ groups+ .into_iter()+ .map(|(name, models)| Ok((name.parse()?, models.unwrap_or_default())))+ .collect()+}++/// Builds an algorithm without provider clients.+/// # Safety+/// JSON must be NUL-terminated and `out` writable. Free the returned handle with `sy_router_free`.+#[unsafe(no_mangle)]+pub unsafe extern "C" fn sy_router_new(+ config_json: *const c_char,+ out: *mut *mut SyRouter,+) -> *mut c_char {+ ffi_call(|| {+ let out = unsafe { out.as_mut() }.ok_or("missing router output")?;+ *out = std::ptr::null_mut();+ let config: RouterConfig = unsafe { parse_json_input(config_json) }?;+ let algorithm = config.algorithm.build("embedded", &config.targets)?;+ let runtime = Runtime::new()?;+ *out = Box::into_raw(Box::new(SyRouter {+ engine: Arc::new(Engine { algorithm, runtime }),+ }));+ Ok(())+ })+}++/// Starts a run with host-approved model groups.+/// # Safety+/// Handles must be live, JSON NUL-terminated, and `out` writable.+#[unsafe(no_mangle)]+pub unsafe extern "C" fn sy_run_new(+ router: *const SyRouter,+ input_json: *const c_char,+ out: *mut *mut SyRun,+) -> *mut c_char {+ ffi_call(|| {+ let router = unsafe { router.as_ref() }.ok_or("missing router")?;+ let out = unsafe { out.as_mut() }.ok_or("missing run output")?;+ *out = std::ptr::null_mut();+ let input: RunInput = unsafe { parse_json_input(input_json) }?;+ let mut models = RuntimeModels::new(parse_model_groups(input.models)?);+ if let Some(subagent_models) = input.subagent_models {+ models = models.with_subagent(parse_model_groups(subagent_models)?);+ }+ let request = Request {+ llm_request: input.request,+ metadata: input.metadata.map(|metadata| Metadata {+ session_id: metadata.session_id,+ agent_id: metadata.agent_id,+ is_subagent: metadata.is_subagent,+ is_delegated_work: metadata.is_delegated_work,+ session_final: metadata.session_final,+ ..Default::default()+ }),+ ..Default::default()+ };+ let _entered = router.engine.runtime.enter();+ let stream = Arc::clone(&router.engine.algorithm).run_stream(+ request,+ Arc::new(models),+ State::default(),+ );+ *out = Box::into_raw(Box::new(SyRun::new(stream, Arc::clone(&router.engine))));+ Ok(())+ })+}++/// Requests the next step asynchronously. Only one read may be pending per run.+/// NULL accepts the read (one callback); an owned error rejects it (no callback).+/// Other runs, model replies, and cancellation can proceed independently.+/// # Safety+/// Keep the run live during this call and the callback state live until its callback returns.+/// Callbacks may overlap or run before this call returns. Free their strings with `sy_string_free`.+/// Callbacks must return promptly without unwinding, calling `sy_run_respond`, or freeing handles.+#[unsafe(no_mangle)]+pub unsafe extern "C" fn sy_run_next_async(+ run: *const SyRun,+ callback: Option<unsafe extern "C" fn(usize, *mut c_char, *mut c_char)>,+ user_data: usize,+) -> *mut c_char {+ ffi_call(|| {+ let run = unsafe { run.as_ref() }.ok_or("missing run")?;+ let callback = callback.ok_or("missing completion callback")?;+ let read = run.state.begin_read()?;+ run.tasks.spawn_on(+ async move {+ let (error, output) = match read.next().await {+ Ok(output) => (std::ptr::null_mut(), output.into_raw()),+ Err(error) => (error_string(error.to_string()), std::ptr::null_mut()),+ };+ unsafe { callback(user_data, error, output) };+ },+ run.engine.runtime.handle(),+ );+ Ok(())+ })+}++/// Delivers a buffered host reply to its pending call.+/// # Safety+/// The run must remain live and JSON must be NUL-terminated. May overlap an async read.+/// Do not call from a completion callback or Tokio task.+#[unsafe(no_mangle)]+pub unsafe extern "C" fn sy_run_respond(+ run: *const SyRun,+ call_id: u64,+ reply_json: *const c_char,+) -> *mut c_char {+ ffi_call(|| {+ let run = unsafe { run.as_ref() }.ok_or("missing run")?;+ let reply: CallReply = unsafe { parse_json_input(reply_json) }?;+ let mut lifecycle = run.state.lifecycle.lock();+ lifecycle.status.require_active()?;+ let call = lifecycle+ .pending+ .remove(&call_id)+ .ok_or("unknown call ID")?;+ let result = match call {+ Call::Model(call) => {+ let response = match reply {+ CallReply::Error { error } => Ok(Err(LibsyError::external(+ "host model call",+ std::io::Error::other(error),+ ))),+ CallReply::Response {+ response,+ served_model,+ } => serde_json::from_value(response).map(|response| {+ Ok(Response {+ llm_response: LlmResponse::Agg(response),+ metadata: served_model.map(|model| Metadata {+ served_model: Some(model),+ ..Default::default()+ }),+ upstream_headers: Default::default(),+ })+ }),+ };+ let response = match response {+ Ok(response) => response,+ Err(error) => {+ lifecycle.pending.insert(call_id, Call::Model(call));+ return Err(error.into());+ }+ };+ run.engine+ .runtime+ .block_on(call.respond(std::future::ready(response)))+ }+ Call::Decision(call) => {+ let response = match reply {+ CallReply::Error { error } => Ok(Err(LibsyError::external(+ "host decision call",+ std::io::Error::other(error),+ ))),+ CallReply::Response { response, .. } => {+ serde_json::from_value(response).map(Ok)+ }+ };+ let response = match response {+ Ok(response) => response,+ Err(error) => {+ lifecycle.pending.insert(call_id, Call::Decision(call));+ return Err(error.into());+ }+ };+ run.engine+ .runtime+ .block_on(call.respond(std::future::ready(response)))+ }+ };+ if result.is_err() {+ lifecycle.status = RunStatus::Failed;+ lifecycle.pending.clear();+ }+ drop(lifecycle);+ if result.is_err() {+ run.state.stop(RunStatus::Failed);+ }+ Ok(result?)+ })+}++/// Signals cancellation without waiting for callbacks.+/// # Safety+/// The run must remain live for the duration of this call.+#[unsafe(no_mangle)]+pub unsafe extern "C" fn sy_run_cancel(run: *const SyRun) {+ let _ = catch_unwind(AssertUnwindSafe(|| {+ if let Some(run) = unsafe { run.as_ref() } {+ run.state.stop(RunStatus::Cancelled);+ }+ }));+}++/// Cancels a run, waits for accepted callbacks to return, and frees it.+/// # Safety+/// Pass a handle from `sy_run_new` once, or NULL. Other calls using it must have returned.+/// Accepted callbacks may still be active; keep their state live until they return.+/// Do not call from a completion callback or Tokio task.+#[unsafe(no_mangle)]+pub unsafe extern "C" fn sy_run_free(run: *mut SyRun) {+ if !run.is_null() {+ let _ = catch_unwind(AssertUnwindSafe(|| drop(unsafe { Box::from_raw(run) })));+ }+}++/// Frees the router handle; existing runs retain its engine.+/// # Safety+/// Pass a handle from `sy_router_new` once, with no concurrent calls using it, or NULL.+#[unsafe(no_mangle)]+pub unsafe extern "C" fn sy_router_free(router: *mut SyRouter) {+ if !router.is_null() {+ let _ = catch_unwind(AssertUnwindSafe(|| drop(unsafe { Box::from_raw(router) })));+ }+}+/// Frees a library-owned error or JSON string./// # Safety/// Pass a returned string once, with no remaining readers, or NULL.diff --git a/crates/switchyard-ffi/src/tests.rs b/crates/switchyard-ffi/src/tests.rsindex 29ec3f397..2cc53ceeb 100644--- a/crates/switchyard-ffi/src/tests.rs+++ b/crates/switchyard-ffi/src/tests.rs@@ -5,6 +5,46 @@use super::*;use serde_json::{Value, json};+use std::sync::atomic::{AtomicUsize, Ordering};+use std::sync::mpsc;+use std::time::Duration;++type Completion = std::result::Result<CString, CString>;++unsafe extern "C" fn complete(user_data: usize, error: *mut c_char, output: *mut c_char) {+ let sender = unsafe { &*(user_data as *const mpsc::Sender<Completion>) };+ let output = (!output.is_null()).then(|| unsafe { CString::from_raw(output) });+ let result = if error.is_null() {+ Ok(output.unwrap_or_default())+ } else {+ Err(unsafe { CString::from_raw(error) })+ };+ let _ = sender.send(result);+}++unsafe extern "C" fn complete_once(user_data: usize, error: *mut c_char, output: *mut c_char) {+ let sender = unsafe { Box::from_raw(user_data as *mut mpsc::Sender<Completion>) };+ unsafe { complete(&*sender as *const _ as usize, error, output) };+}++fn next_async(run: &SyRun, sender: &mpsc::Sender<Completion>) -> std::result::Result<(), String> {+ let sender = Box::into_raw(Box::new(sender.clone()));+ let error = unsafe { sy_run_next_async(run, Some(complete_once), sender as usize) };+ if error.is_null() {+ Ok(())+ } else {+ drop(unsafe { Box::from_raw(sender) });+ Err(error_message(error))+ }+}++fn received_step(receiver: &mpsc::Receiver<Completion>) -> Value {+ let output = receiver+ .recv_timeout(Duration::from_secs(5))+ .unwrap()+ .unwrap();+ serde_json::from_slice(output.as_bytes()).unwrap()+}// ABI generations are independent of package releases.#[test]@@ -36,6 +76,37 @@ fn check(error: *mut c_char) {}}+fn error_message(error: *mut c_char) -> String {+ assert!(!error.is_null(), "expected an error");+ unsafe { CString::from_raw(error) }.into_string().unwrap()+}++fn router(algorithm: Value) -> Box<SyRouter> {+ let config = json!({+ "algorithm": algorithm,+ "targets": {"capable": "capable", "efficient": "efficient", "judge": "judge"},+ });+ let config = CString::new(config.to_string()).unwrap();+ let mut router = std::ptr::null_mut();+ check(unsafe { sy_router_new(config.as_ptr(), &mut router) });+ assert!(!router.is_null());+ unsafe { Box::from_raw(router) }+}++fn run(router: &SyRouter, input: Value) -> Box<SyRun> {+ let input = CString::new(input.to_string()).unwrap();+ let mut run = std::ptr::null_mut();+ check(unsafe { sy_run_new(router, input.as_ptr(), &mut run) });+ assert!(!run.is_null());+ unsafe { Box::from_raw(run) }+}++fn next(run: &SyRun) -> Value {+ let (sender, receiver) = mpsc::channel();+ next_async(run, &sender).unwrap();+ received_step(&receiver)+}+fn translated(call: unsafe extern "C" fn(*const c_char, *mut *mut c_char) -> *mut c_char,input: Value,@@ -49,6 +120,224 @@ fn translated(result.unwrap()}+fn advisor_router() -> Box<SyRouter> {+ router(json!({+ "type": "advisor", "executor_target": "capable", "advisor_target": "judge",+ "gate_trigger": "pattern", "gate_trigger_pattern": "NEVER",+ }))+}++fn advisor_input() -> Value {+ json!({"request": {}, "models": {"efficient": ["capable"], "judge": ["judge"]}})+}++fn pending_advisor() -> (Box<SyRun>, u64) {+ let router = advisor_router();+ let run = run(&router, advisor_input());+ let call = next(&run);+ assert_eq!(call["type"], "call_model");+ (run, call["id"].as_u64().unwrap())+}++#[test]+fn completed_runs_reject_further_reads() {+ let router = router(json!({"type": "passthrough", "target": "capable"}));+ let run = run(+ &router,+ json!({"request": {}, "models": {"any": ["capable"]}}),+ );+ unsafe { sy_router_free(Box::into_raw(router)) };+ assert_eq!(next(&run)["type"], "done");+ let (sender, receiver) = mpsc::channel();+ for _ in 0..2 {+ assert_eq!(+ next_async(&run, &sender).unwrap_err(),+ "routing already completed"+ );+ assert!(matches!(+ receiver.try_recv(),+ Err(mpsc::TryRecvError::Empty)+ ));+ }+ unsafe { sy_run_free(Box::into_raw(run)) };+}++// Streamed answers still use the buffered Done contract without losing the answer.+#[test]+fn noop_outcomes_return_buffered_answers() {+ let router = router(json!({"type": "noop"}));+ for stream in [false, true] {+ let run = run(+ &router,+ json!({+ "request": {"model": "capable", "stream": stream},+ "models": {"any": ["capable"]},+ }),+ );+ let done = next(&run);+ assert_eq!(done["type"], "done");+ assert_eq!(done["selected_model_ids"], json!(["capable"]));+ assert_eq!(done["request"]["model"], "capable");+ assert_eq!(done["request"]["stream"], stream);+ assert_eq!(done["response"]["id"], "switchyard-noop");+ assert_eq!(done["response"]["model"], "capable");+ assert_eq!(done["response"]["outputs"][0]["content"][0]["text"], "OK");+ }+}++// Cancellation must interrupt response aggregation, not just waiting for a routing step.+#[test]+fn streamed_outcomes_propagate_errors_and_cancellation() {+ let router = router(json!({"type": "noop"}));+ for (cancel, expected_error, terminal_error) in [+ (false, "stream failed", "routing failed"),+ (true, "routing cancelled", "routing cancelled"),+ ] {+ let (polled, started) = mpsc::channel();+ let response = Response {+ llm_response: LlmResponse::Stream(Box::pin(futures::stream::poll_fn(move |_| {+ let _ = polled.send(());+ if cancel {+ std::task::Poll::Pending+ } else {+ std::task::Poll::Ready(Some(Err(switchyard_protocol::LlmClientError::General(+ "stream failed".into(),+ ))))+ }+ }))),+ metadata: None,+ upstream_headers: Default::default(),+ };+ let outcome = switchyard_libsy::RoutingOutcome::answered(+ "capable".into(),+ Request::default(),+ response,+ );+ let run = Box::new(SyRun::new(+ Box::pin(futures::stream::iter([Ok(Step::Done(Box::new(outcome)))])),+ Arc::clone(&router.engine),+ ));+ let (sender, receiver) = mpsc::channel();+ next_async(&run, &sender).unwrap();+ started.recv_timeout(Duration::from_secs(5)).unwrap();+ if cancel {+ unsafe { sy_run_cancel(&*run) };+ }+ let error = receiver+ .recv_timeout(Duration::from_secs(5))+ .unwrap()+ .unwrap_err();+ assert_eq!(error.to_str().unwrap(), expected_error);+ assert_eq!(next_async(&run, &sender).unwrap_err(), terminal_error);+ unsafe { sy_run_free(Box::into_raw(run)) };+ }+}++// Fallback candidates retain their request when the selected model's request was rewritten.+#[test]+fn completed_runs_preserve_fallback_requests() {+ let router = router(json!({"type": "passthrough", "target": "capable"}));+ for fallback in [+ Some(switchyard_protocol::text_request(None, "original")),+ None,+ ] {+ let expected = serde_json::to_value(&fallback).unwrap();+ let mut outcome = switchyard_libsy::RoutingOutcome::route_to(+ "capable".into(),+ vec!["efficient".into()],+ Request {+ llm_request: switchyard_protocol::text_request(None, "rewritten"),+ ..Default::default()+ },+ );+ outcome.fallback_request = fallback.map(|llm_request| Request {+ llm_request,+ ..Default::default()+ });+ let run = SyRun::new(+ Box::pin(futures::stream::iter([Ok(Step::Done(Box::new(outcome)))])),+ Arc::clone(&router.engine),+ );+ let done = next(&run);+ assert_eq!(done["selected_model_ids"], json!(["capable", "efficient"]));+ assert_eq!(+ done["request"]["messages"][0]["content"][0]["text"],+ "rewritten"+ );+ assert_eq!(done.get("fallback_request"), Some(&expected));+ }+}++#[test]+fn cancellation_completes_pending_read_and_rejects_replies() {+ let (run, id) = pending_advisor();+ let (sender, receiver) = mpsc::channel();+ next_async(&run, &sender).unwrap();+ assert_eq!(+ next_async(&run, &sender).unwrap_err(),+ "routing read already in progress"+ );+ unsafe { sy_run_cancel(&*run) };+ let error = receiver+ .recv_timeout(Duration::from_secs(5))+ .unwrap()+ .unwrap_err();+ assert_eq!(error.to_str().unwrap(), "routing cancelled");+ let error = unsafe { sy_run_respond(&*run, id, c"{\"response\":{}}".as_ptr()) };+ assert_eq!(error_message(error), "routing cancelled");+}++// A malformed reply must not consume its call ID.+#[test]+fn invalid_model_replies_preserve_pending_calls() {+ let (run, id) = pending_advisor();+ let error = unsafe {+ sy_run_respond(+ &*run,+ id,+ c"{\"response\":{\"outputs\":\"wrong\"}}".as_ptr(),+ )+ };+ assert!(!error_message(error).is_empty());+ check(unsafe { sy_run_respond(&*run, id, c"{\"response\":{}}".as_ptr()) });+ let error = unsafe { sy_run_respond(&*run, id, c"{\"response\":{}}".as_ptr()) };+ assert_eq!(error_message(error), "unknown call ID");+ assert_eq!(next(&run)["type"], "done");+}++// Decision replies have a different schema, but the same retryable validation boundary.+#[test]+fn invalid_decision_replies_preserve_pending_calls() {+ let router = router(json!({+ "type": "llm_classifier", "classifier_target": "judge",+ "strong_target": "capable", "weak_target": "efficient", "fail_open": false,+ "decision": {"cutoff": 0.5, "candidates": {"a": "capable", "b": "efficient"}, "evidence": {}},+ }));+ let run = run(+ &router,+ json!({+ "request": {},+ "models": {+ "any": ["capable", "efficient"], "capable": ["capable"],+ "efficient": ["efficient"], "judge": ["judge"],+ },+ }),+ );+ let call = next(&run);+ assert_eq!(call["type"], "call_decision");+ assert_eq!(call["model"], "judge");+ let id = call["id"].as_u64().unwrap();+ let error = unsafe { sy_run_respond(&*run, id, c"{\"response\":{}}".as_ptr()) };+ assert!(!error_message(error).is_empty());+ let reply = CString::new(json!({"response": {"answers": {"route": {"value": {+ "type": "choice", "data": {"selected": "advantage", "probabilities": {"advantage": 0.8, "no_advantage": 0.2}},+ }}}}}).to_string()).unwrap();+ check(unsafe { sy_run_respond(&*run, id, reply.as_ptr()) });+ let done = next(&run);+ assert_eq!(done["type"], "done");+ assert_eq!(done["selected_model_ids"], json!(["capable", "efficient"]));+}+// Translation exports return owned JSON strings that the caller releases through the ABI.#[test]fn translation_exports_return_owned_json() {@@ -76,3 +365,264 @@ fn translation_exports_return_owned_json() {);assert_eq!(response["outputs"][0]["content"][0]["text"], "hello");}++// Pending reads must yield the runtime worker, not consume blocking threads.+#[test]+fn pending_reads_yield_runtime_worker() {+ let thread_starts = Arc::new(AtomicUsize::new(0));+ let observed = Arc::clone(&thread_starts);+ let mut router = advisor_router();+ Arc::get_mut(&mut router.engine).unwrap().runtime = tokio::runtime::Builder::new_multi_thread()+ .worker_threads(1)+ .enable_all()+ .on_thread_start(move || {+ observed.fetch_add(1, Ordering::SeqCst);+ })+ .build()+ .unwrap();+ let (sender, receiver) = mpsc::channel();+ let mut runs = Vec::new();+ for _ in 0..32 {+ let run = run(&router, advisor_input());+ next_async(&run, &sender).unwrap();+ assert_eq!(received_step(&receiver)["type"], "call_model");+ next_async(&run, &sender).unwrap();+ runs.push(run);+ }+ let (scheduled, sentinel) = mpsc::channel();+ router.engine.runtime.spawn(async move {+ let _ = scheduled.send(());+ });+ sentinel.recv_timeout(Duration::from_secs(5)).unwrap();+ assert_eq!(thread_starts.load(Ordering::SeqCst), 1);+ drop(runs);+ for _ in 0..32 {+ let error = receiver+ .recv_timeout(Duration::from_secs(5))+ .unwrap()+ .unwrap_err();+ assert_eq!(error.to_str().unwrap(), "routing cancelled");+ }+}++struct BlockedCompletion {+ sender: mpsc::Sender<Completion>,+ resume: Mutex<mpsc::Receiver<()>>,+ finished: mpsc::Sender<()>,+}++unsafe extern "C" fn complete_after_release(+ user_data: usize,+ error: *mut c_char,+ output: *mut c_char,+) {+ let state = unsafe { &*(user_data as *const BlockedCompletion) };+ unsafe { complete(&state.sender as *const _ as usize, error, output) };+ let _ = state.resume.lock().recv();+ let _ = state.finished.send(());+}++// Keep another runtime owner so runtime shutdown cannot mask an early free.+#[test]+fn free_waits_for_cancelled_callback_to_return() {+ let engine = Arc::new(Engine {+ algorithm: Arc::new(switchyard_libsy::Noop {}),+ runtime: tokio::runtime::Builder::new_multi_thread()+ .worker_threads(2)+ .enable_all()+ .build()+ .unwrap(),+ });+ let (sender, receiver) = mpsc::channel();+ let (resume, paused) = mpsc::channel();+ let (finished, returned) = mpsc::channel();+ let state = Box::into_raw(Box::new(BlockedCompletion {+ sender,+ resume: Mutex::new(paused),+ finished,+ }));+ let run = Box::new(SyRun::new(+ Box::pin(futures::stream::pending()),+ Arc::clone(&engine),+ ));+ check(unsafe { sy_run_next_async(&*run, Some(complete_after_release), state as usize) });+ let run = Box::into_raw(run) as usize;+ let (freed, closed) = mpsc::channel();+ let freeing = std::thread::spawn(move || {+ unsafe { sy_run_free(run as *mut SyRun) };+ let _ = freed.send(());+ });+ let cancellation = receiver.recv_timeout(Duration::from_secs(5));+ let premature_free = closed.recv_timeout(Duration::from_millis(50));+ let _ = resume.send(());+ returned.recv_timeout(Duration::from_secs(5)).unwrap();+ if premature_free.is_err() {+ closed.recv_timeout(Duration::from_secs(5)).unwrap();+ }+ freeing.join().unwrap();+ drop(unsafe { Box::from_raw(state) });+ assert!(matches!(+ premature_free,+ Err(mpsc::RecvTimeoutError::Timeout)+ ));+ assert_eq!(+ cancellation.unwrap().unwrap_err().to_str().unwrap(),+ "routing cancelled"+ );+}++// Runs may share call IDs without sharing replies or cancellation.+#[test]+fn async_runs_complete_independently() {+ let router = advisor_router();+ let (sender, receiver) = mpsc::channel();+ let runs: Vec<_> = (0..3).map(|_| run(&router, advisor_input())).collect();+ let mut ids = Vec::new();+ for run in &runs {+ next_async(run, &sender).unwrap();+ ids.push(received_step(&receiver)["id"].as_u64().unwrap());+ next_async(run, &sender).unwrap();+ }+ for (index, text) in [(1, "second"), (0, "first")] {+ if index == 0 {+ unsafe { sy_run_cancel(&*runs[2]) };+ let error = receiver+ .recv_timeout(Duration::from_secs(5))+ .unwrap()+ .unwrap_err();+ assert_eq!(error.to_str().unwrap(), "routing cancelled");+ }+ let reply = CString::new(+ json!({+ "response": switchyard_protocol::text_response(None, text),+ })+ .to_string(),+ )+ .unwrap();+ check(unsafe { sy_run_respond(&*runs[index], ids[index], reply.as_ptr()) });+ let done = received_step(&receiver);+ assert_eq!(done["type"], "done");+ assert_eq!(done["response"]["outputs"][0]["content"][0]["text"], text);+ }+}++// Host fallback attribution uses target IDs, not provider model names.+#[test]+fn advisor_attributes_host_fallback() {+ let router = advisor_router();+ for served_model in [None, Some("fallback")] {+ let input = json!({+ "request": {}, "models": {"efficient": ["capable", "fallback"], "judge": ["judge"]},+ });+ let run = run(&router, input);+ let call = next(&run);+ assert_eq!(call["models"], json!(["capable", "fallback"]));+ let mut reply = json!({+ "response": switchyard_protocol::text_response(Some("provider-model".into()), "answer"),+ });+ if let Some(model) = served_model {+ reply["served_model"] = json!(model);+ }+ let reply = CString::new(reply.to_string()).unwrap();+ check(unsafe { sy_run_respond(&*run, call["id"].as_u64().unwrap(), reply.as_ptr()) });+ let done = next(&run);+ let expected = served_model.unwrap_or("capable");+ assert_eq!(done["type"], "done");+ assert_eq!(done["selected_model_ids"], json!([expected]));+ assert_eq!(done["request"]["model"], expected);+ assert_eq!(done["response"]["model"], "provider-model");+ }+}++// Child sessions keep separate phases and clear them after the final request.+#[test]+fn plan_execute_isolates_and_finishes_child_sessions() {+ let router = router(json!({+ "type": "plan_execute", "capable_target": "capable", "efficient_target": "efficient",+ }));+ for (agent, is_subagent, is_final, edited, expected) in [+ ("a", true, false, true, "efficient"),+ ("b", true, false, false, "capable"),+ ("root", false, false, false, "capable"),+ ("a", true, true, false, "efficient"),+ ("a", true, false, false, "capable"),+ ] {+ let request = if edited {+ json!({"messages": [{"role": "assistant", "content": [{+ "type": "tool_call", "id": "edit", "name": "apply_patch", "arguments": {},+ }]}]})+ } else {+ json!({})+ };+ let input = json!({+ "request": request, "models": {"capable": ["capable"], "efficient": ["efficient"]},+ "metadata": {"session_id": "s", "agent_id": agent, "is_subagent": is_subagent,+ "session_final": is_final},+ });+ let run = run(&router, input);+ assert_eq!(+ next(&run)["selected_model_ids"],+ json!([expected]),+ "{agent}"+ );+ }+}++// Child identity alone must not activate the delegated-work policy.+#[test]+fn delegated_work_uses_subagent_model_groups() {+ let router = router(json!({+ "type": "passthrough", "target": "capable",+ "subagents": {"type": "passthrough", "target": "efficient"},+ }));+ for (metadata, expected) in [+ (Value::Null, "capable"),+ (json!({"is_subagent": true}), "capable"),+ (+ json!({"is_subagent": true, "is_delegated_work": true}),+ "efficient",+ ),+ ] {+ let input = json!({+ "request": {}, "models": {"any": ["capable"]},+ "subagent_models": {"any": ["efficient"]}, "metadata": metadata,+ });+ let run = run(&router, input);+ assert_eq!(next(&run)["selected_model_ids"], json!([expected]));+ }+}++#[test]+fn invalid_router_input_returns_owned_errors() {+ for input in [std::ptr::null(), c"{".as_ptr()] {+ let mut router = std::ptr::dangling_mut::<SyRouter>();+ let error = unsafe { sy_router_new(input, &mut router) };+ assert!(!error.is_null());+ assert!(router.is_null());+ let message = unsafe { CStr::from_ptr(error) }.to_str().unwrap();+ assert!(!message.is_empty());+ unsafe { sy_string_free(error) };+ }+ let error = unsafe { sy_router_new(c"{".as_ptr(), std::ptr::null_mut()) };+ assert!(!error.is_null());+ unsafe { sy_string_free(error) };+}++// Unwinding panics become owned errors at the C boundary.+#[test]+fn panics_return_owned_errors() {+ let run = SyRun::new(+ Box::pin(futures::stream::poll_fn(|_| panic!("test panic"))),+ Arc::new(Engine {+ algorithm: Arc::new(switchyard_libsy::Noop {}),+ runtime: Runtime::new().unwrap(),+ }),+ );+ let (sender, receiver) = mpsc::channel();+ next_async(&run, &sender).unwrap();+ let error = receiver+ .recv_timeout(Duration::from_secs(5))+ .unwrap()+ .unwrap_err();+ assert_eq!(error.to_str().unwrap(), "Switchyard binding panicked");+}