Skip to content

Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.

Rust

chore: Rename `CallLlm` to `CallModel` (#340)

Graham King94872ce5923c
diff --git a/crates/libsy-llm-client/src/run.rs b/crates/libsy-llm-client/src/run.rs
index 2b8e017c4..092a7b04a 100644
--- a/crates/libsy-llm-client/src/run.rs
+++ b/crates/libsy-llm-client/src/run.rs
@@ -16,7 +16,7 @@ use std::sync::Arc;
use std::time::{Duration, Instant};
use parking_lot::Mutex;
-use switchyard_libsy::{Algorithm, CallLlmRequest, LibsyError, Result, algorithm_label, drive};
+use switchyard_libsy::{Algorithm, CallModelRequest, LibsyError, Result, algorithm_label, drive};
use switchyard_protocol::{Context, Decision, LlmClientError, Request, Response, RoutedLlmClient};
use crate::observation::{LlmCallObservation, RunObservation, RunObserver};
@@ -141,7 +141,7 @@ impl RoutedCallWindows {
)]
async fn serve(
clients: ClientRouter,
- call: CallLlmRequest,
+ call: CallModelRequest,
observer: Option<RunObserver>,
// Output parameter because `drive` takes a function that returns a plain `Result<()>`.
routed_calls: Arc<Mutex<RoutedCallWindows>>,
diff --git a/crates/libsy-llm-client/tests/observability.rs b/crates/libsy-llm-client/tests/observability.rs
index 84630ee65..7c7865120 100644
--- a/crates/libsy-llm-client/tests/observability.rs
+++ b/crates/libsy-llm-client/tests/observability.rs
@@ -458,7 +458,7 @@ impl Algorithm for SingleCallAlgo {
));
driver.info(ctx.clone(), decision.clone()).await?;
driver
- .call_llm(RoutedRequest {
+ .call_model(RoutedRequest {
request,
decision,
ctx,
@@ -1039,7 +1039,7 @@ async fn failed_call_records_error_outcome_and_warn_logs() -> switchyard_libsy::
let mut saw_error_step = false;
while let Some(step) = stream.next().await {
match step {
- Ok(Step::CallLlm(call)) => {
+ Ok(Step::CallModel(call)) => {
call.respond(Err(test_error("synthetic upstream failure")))?;
}
Ok(Step::Decision(_)) => {}
diff --git a/crates/libsy/README.md b/crates/libsy/README.md
index 9f0f95aa4..5a2ea5638 100644
--- a/crates/libsy/README.md
+++ b/crates/libsy/README.md
@@ -33,7 +33,7 @@ tokio = { version = "1", features = ["macros", "rt"] }
[`LlmTarget`] names a routing destination. An [`Algorithm`] selects targets and
records [`Decision`](switchyard_protocol::Decision)s, offloading every model call
to its caller: [`Algorithm::run_stream`] yields a [`Step`] stream whose
-[`Step::CallLlm`] items the host serves over its own transport. libsy makes no
+[`Step::CallModel`] items the host serves over its own transport. libsy makes no
network calls itself — `switchyard-llm-client`'s `run` is a ready-made consumer
that drives the stream and performs the calls over HTTP.
diff --git a/crates/libsy/src/algorithms/fall_through.rs b/crates/libsy/src/algorithms/fall_through.rs
index a290a0a12..4c44c571c 100644
--- a/crates/libsy/src/algorithms/fall_through.rs
+++ b/crates/libsy/src/algorithms/fall_through.rs
@@ -230,7 +230,7 @@ where
match served {
Some(response) => Ok(response),
None => {
- algorithm::call_llm_with_fallback(
+ algorithm::call_model_with_fallback(
ctx,
&driver,
&self.targets,
diff --git a/crates/libsy/src/algorithms/llm_class.rs b/crates/libsy/src/algorithms/llm_class.rs
index 1df07ae06..725d91c88 100644
--- a/crates/libsy/src/algorithms/llm_class.rs
+++ b/crates/libsy/src/algorithms/llm_class.rs
@@ -517,9 +517,9 @@ impl Classifier<State> for EscalationClassifier {
//
// If the efficient model exceeds its context window, fall through to capable: returning
// `(decisive(capable), None)` tells FallThrough::execute to call
- // call_llm_with_fallback with the capable target instead of surfacing the error.
+ // call_model_with_fallback with the capable target instead of surfacing the error.
let efficient_response = match driver
- .call_llm(RoutedRequest {
+ .call_model(RoutedRequest {
request: request.clone(),
decision: Arc::new(Decision::new(
self.efficient.semantic_name.clone(),
diff --git a/crates/libsy/src/algorithms/passthrough.rs b/crates/libsy/src/algorithms/passthrough.rs
index 5a2a93284..cab9140bd 100644
--- a/crates/libsy/src/algorithms/passthrough.rs
+++ b/crates/libsy/src/algorithms/passthrough.rs
@@ -45,7 +45,7 @@ impl Algorithm for Passthrough {
));
driver.info(ctx.clone(), decision.clone()).await?;
driver
- .call_llm(RoutedRequest {
+ .call_model(RoutedRequest {
request,
decision,
ctx,
diff --git a/crates/libsy/src/algorithms/util/llm_judge.rs b/crates/libsy/src/algorithms/util/llm_judge.rs
index e4a75a018..ae47123d7 100644
--- a/crates/libsy/src/algorithms/util/llm_judge.rs
+++ b/crates/libsy/src/algorithms/util/llm_judge.rs
@@ -227,7 +227,7 @@ where
let judge_model = self.target.semantic_name.as_str();
let response = driver
- .call_llm(RoutedRequest {
+ .call_model(RoutedRequest {
request: self.judge.build_request(state, request),
decision: Arc::new(Decision::new(
self.target.semantic_name.to_string(),
@@ -470,7 +470,7 @@ mod tests {
let mut request = request();
let serve = async {
- if let Some(Ok(Step::CallLlm(call))) = steps.next().await {
+ if let Some(Ok(Step::CallModel(call))) = steps.next().await {
let _ = call.respond(reply);
}
};
diff --git a/crates/libsy/src/core/algorithm.rs b/crates/libsy/src/core/algorithm.rs
index 73270a7cb..43820d5e1 100644
--- a/crates/libsy/src/core/algorithm.rs
+++ b/crates/libsy/src/core/algorithm.rs
@@ -39,7 +39,7 @@ use crate::{DriverError, LibsyError, Result, observability};
pub type StepStream = Pin<Box<dyn Stream<Item = Result<Step>> + Send>>;
/// A request paired with the routing [`Decision`] that produced it — the offload
-/// payload a host reads (via [`CallLlmRequest::get_routed`]) to serve the call.
+/// payload a host reads (via [`CallModelRequest::get_routed`]) to serve the call.
///
/// The selected model and inbound route name live in separate, unambiguous places: the model
/// identifier is [`decision.selected_model_id()`](Decision::selected_model_id), while
@@ -57,19 +57,19 @@ pub struct RoutedRequest {
pub ctx: Context,
}
-/// The host-facing half of an offloaded model call, surfaced inside [`Step::CallLlm`].
+/// The host-facing half of an offloaded model call, surfaced inside [`Step::CallModel`].
///
/// The host reads the routed request ([`get_routed`](Self::get_routed)) and the decision
/// behind it ([`get_decision`](Self::get_decision)), performs (or delegates) the model
/// call, and fulfills it with [`respond`](Self::respond) — unblocking the algorithm's
-/// [`Driver::call_llm`] on the other side. `switchyard-llm-client`'s `run` is the ready-made
+/// [`Driver::call_model`] on the other side. `switchyard-llm-client`'s `run` is the ready-made
/// consumer that does this for you.
-pub struct CallLlmRequest {
+pub struct CallModelRequest {
routed: RoutedRequest,
reply: oneshot::Sender<Result<Response>>,
}
-impl CallLlmRequest {
+impl CallModelRequest {
/// The routed request the host should serve; its `decision.selected_model_id()` names
/// the model to hit.
pub fn get_routed(&self) -> &RoutedRequest {
@@ -98,7 +98,7 @@ impl CallLlmRequest {
/// The offload channel handed to an algorithm's
/// [`create_run_task`](Algorithm::create_run_task). The algorithm makes model calls
-/// with [`call_llm`](Self::call_llm) and publishes its [`Decision`]s with
+/// with [`call_model`](Self::call_model) and publishes its [`Decision`]s with
/// [`info`](Self::info); each call is offloaded to the request's [`Step`] stream and
/// awaits the consumer's response. The step channel is bounded, so the consumer paces
/// the algorithm one step at a time.
@@ -119,7 +119,7 @@ impl Driver {
(Self { step_tx }, step_rx)
}
- /// Offload a model call: publish `routed` as a [`Step::CallLlm`] and await the
+ /// Offload a model call: publish `routed` as a [`Step::CallModel`] and await the
/// consumer's [`Response`]. The call's context travels inside
/// [`routed.ctx`](RoutedRequest::ctx). Errors if the stream is closed or the call failed.
/// The await is wrapped in a `libsy.llm_call` span measuring *fulfillment* as
@@ -143,16 +143,16 @@ impl Driver {
reasoning_tokens = tracing::field::Empty,
)
)]
- pub async fn call_llm(&self, routed: RoutedRequest) -> Result<Response> {
+ pub async fn call_model(&self, routed: RoutedRequest) -> Result<Response> {
let algorithm = observability::algorithm_label(&routed.ctx).to_string();
let decision = Arc::clone(&routed.decision);
let is_answer_call = decision.is_answer_call();
let started = Instant::now();
let (reply, response) = oneshot::channel::<Result<Response>>();
- let call = CallLlmRequest { routed, reply };
+ let call = CallModelRequest { routed, reply };
let result = async {
self.step_tx
- .send(Ok(Step::CallLlm(Box::new(call))))
+ .send(Ok(Step::CallModel(Box::new(call))))
.await
.map_err(|_| DriverError::StreamClosed)?;
response
@@ -199,8 +199,8 @@ impl Driver {
/// One item in the stream returned by [`Algorithm::run_stream`].
pub enum Step {
/// The algorithm needs this model call performed. The host serves it and fulfills
- /// it with [`CallLlmRequest::respond`]. Boxed: it is by far the largest variant.
- CallLlm(Box<CallLlmRequest>),
+ /// it with [`CallModelRequest::respond`]. Boxed: it is by far the largest variant.
+ CallModel(Box<CallModelRequest>),
/// A routing decision the algorithm made, published via [`Driver::info`] as it
/// happens (rather than collected into a trace returned at the end).
Decision(Arc<Decision>),
@@ -212,7 +212,7 @@ pub enum Step {
///
/// Returns the final [`Response`] and the trace of [`Decision`]s the algorithm published.
/// `serve` owns the call: it performs it however the host likes and must fulfill the promise
-/// with [`CallLlmRequest::respond`]. A failed *model* call belongs in `respond` — the
+/// with [`CallModelRequest::respond`]. A failed *model* call belongs in `respond` — the
/// algorithm may route around it. Returning `Err` from `serve` aborts the whole run, so
/// reserve it for infrastructure failures. Calls are served concurrently, so an algorithm
/// that offloads several at once (hedging, fan-out) gets real parallelism.
@@ -227,7 +227,7 @@ pub async fn drive<F, Fut>(
serve: F,
) -> Result<(Vec<Arc<Decision>>, Response)>
where
- F: Fn(CallLlmRequest) -> Fut,
+ F: Fn(CallModelRequest) -> Fut,
Fut: Future<Output = Result<()>>,
{
let stream = algorithm.run_stream(ctx, request);
@@ -240,14 +240,14 @@ where
loop {
tokio::select! {
Some(result) = in_flight.next() => match result {
- Ok(()) => {}, // CallLlm completed successfully
- Err(err) => return Err(err), // CallLlm failed, propagate the error
+ Ok(()) => {}, // CallModel completed successfully
+ Err(err) => return Err(err), // CallModel failed, propagate the error
},
step = stream.next() => {
match step {
None => break, // stream has ended, no more steps
Some(item) => match item? {
- Step::CallLlm(call) => in_flight.push(serve(*call)),
+ Step::CallModel(call) => in_flight.push(serve(*call)),
Step::Decision(decision) => trace.push(decision),
Step::ReturnToAgent(response) => {
final_response = Some(*response);
@@ -473,7 +473,7 @@ fn classify_fallback(error: &LibsyError) -> Option<(&str, RoutingFallbackReason)
/// `fallback_decision` builds the [`Decision`] published for a `from -> to` hop. Context
/// overflows are recorded for `identity`; unavailable targets remain request-local.
#[allow(clippy::too_many_arguments)]
-pub(crate) async fn call_llm_with_fallback(
+pub(crate) async fn call_model_with_fallback(
mut ctx: Context,
driver: &Driver,
targets: &LlmTargetSet,
@@ -487,7 +487,7 @@ pub(crate) async fn call_llm_with_fallback(
) -> Result<Response> {
loop {
let result = driver
- .call_llm(RoutedRequest {
+ .call_model(RoutedRequest {
request: request.clone(),
decision: decision.clone(),
ctx: ctx.clone(),
@@ -516,7 +516,7 @@ pub(crate) async fn call_llm_with_fallback(
}
/// An optimization strategy. Implement [`create_run_task`](Self::create_run_task);
-/// callers drive it with [`run_stream`](Self::run_stream), serving each [`Step::CallLlm`]
+/// callers drive it with [`run_stream`](Self::run_stream), serving each [`Step::CallModel`]
/// it emits. `switchyard-llm-client`'s `run` is the ready-made consumer that does this
/// over HTTP.
///
@@ -543,7 +543,7 @@ pub trait Algorithm: Send + Sync + 'static {
/// emits for its runs.
fn name(&self) -> &str;
- /// Run one request to completion: make model calls with [`Driver::call_llm`],
+ /// Run one request to completion: make model calls with [`Driver::call_model`],
/// publish [`Decision`]s with [`Driver::info`], and return the final [`Response`].
/// The method an algorithm implements; [`run_stream`](Self::run_stream) drives it.
/// `ctx` carries the request's cross-cutting values (today: the algorithm's
@@ -565,7 +565,7 @@ pub trait Algorithm: Send + Sync + 'static {
/// Process a request to completion, returning a stream of [`Step`]s.
///
- /// The consumer must fulfill every [`Step::CallLlm`] before the algorithm can
+ /// The consumer must fulfill every [`Step::CallModel`] before the algorithm can
/// continue. The bounded step channel applies backpressure when the consumer is
/// not polling. A successful run ends with [`Step::ReturnToAgent`]; a failure is
/// emitted as an `Err` item. Dropping the stream aborts the spawned algorithm task.
@@ -725,7 +725,7 @@ mod tests {
let decision = test_decision(target.semantic_name.clone());
driver.info(ctx.clone(), decision.clone()).await?;
driver
- .call_llm(RoutedRequest {
+ .call_model(RoutedRequest {
request,
decision,
ctx,
@@ -773,14 +773,14 @@ mod tests {
let (driver, mut step_rx) = Driver::new();
let first_driver = driver.clone();
let mut first =
- tokio::spawn(async move { first_driver.call_llm(routed("first")).await });
- let second = tokio::spawn(async move { driver.call_llm(routed("second")).await });
+ tokio::spawn(async move { first_driver.call_model(routed("first")).await });
+ let second = tokio::spawn(async move { driver.call_model(routed("second")).await });
let mut calls = HashMap::new();
for _ in 0..2 {
let step = step_rx.recv().await.ok_or(DriverError::StreamClosed)??;
- let Step::CallLlm(call) = step else {
- return Err(test_error("expected a CallLlm step"));
+ let Step::CallModel(call) = step else {
+ return Err(test_error("expected a CallModel step"));
};
calls.insert(call.get_decision().selected_model_id().to_string(), call);
}
@@ -816,10 +816,10 @@ mod tests {
// Dropping the host-facing promise closes only that call's reply channel.
let (driver, mut step_rx) = Driver::new();
- let producer = tokio::spawn(async move { driver.call_llm(routed("dropped")).await });
+ let producer = tokio::spawn(async move { driver.call_model(routed("dropped")).await });
let step = step_rx.recv().await.ok_or(DriverError::StreamClosed)??;
- let Step::CallLlm(call) = step else {
- return Err(test_error("expected a CallLlm step"));
+ let Step::CallModel(call) = step else {
+ return Err(test_error("expected a CallModel step"));
};
drop(call);
let result = producer
@@ -932,7 +932,7 @@ mod tests {
#[tokio::test]
async fn run_offloads_via_promise_then_returns_to_agent() -> Result<()> {
// Every call is offloaded via a promise the orchestrator surfaces as a
- // `CallLlm` step for us to fulfill.
+ // `CallModel` step for us to fulfill.
let stream = orch(target_set(&["offload/model"])).run_stream(Context::default(), request());
tokio::pin!(stream);
@@ -940,7 +940,7 @@ mod tests {
let mut final_completion = None;
while let Some(step) = stream.next().await {
match step? {
- Step::CallLlm(call) => {
+ Step::CallModel(call) => {
saw_call = true;
// The decision rode along with the promise.
assert_eq!(call.get_decision().selected_model_id(), "offload/model");
@@ -968,7 +968,7 @@ mod tests {
}
}
- assert!(saw_call, "expected a CallLlm step before ReturnToAgent");
+ assert!(saw_call, "expected a CallModel step before ReturnToAgent");
assert_eq!(
final_completion.ok_or_else(|| test_error("no ReturnToAgent step"))?,
"fulfilled"
@@ -1051,7 +1051,7 @@ mod tests {
#[tokio::test]
async fn offload_error_propagates_back_to_the_algorithm() -> Result<()> {
// A client-less target offloads its call; we fulfill the promise with an
- // Err, which must flow back through `call_llm_target` into the algorithm and
+ // Err, which must flow back through `call_model_target` into the algorithm and
// out as an error step — not a response.
let stream = orch(target_set(&["offload/model"])).run_stream(Context::default(), request());
tokio::pin!(stream);
@@ -1059,7 +1059,7 @@ mod tests {
let mut saw_error = false;
while let Some(step) = stream.next().await {
match step {
- Ok(Step::CallLlm(call)) => {
+ Ok(Step::CallModel(call)) => {
call.respond(Err(test_error("upstream model call failed")))?;
}
Ok(Step::Decision(_)) => {}
@@ -1069,7 +1069,7 @@ mod tests {
));
}
Err(err) => {
- // The algorithm's `call_llm_target` saw the error via the promise.
+ // The algorithm's `call_model_target` saw the error via the promise.
assert!(err.to_string().contains("upstream model call failed"));
saw_error = true;
}
@@ -1308,12 +1308,12 @@ mod tests {
) -> Result<Response> {
let dec_w = test_decision(self.winner.semantic_name.clone());
let dec_l = test_decision(self.loser.semantic_name.clone());
- let win = driver.call_llm(RoutedRequest {
+ let win = driver.call_model(RoutedRequest {
request: request.clone(),
decision: dec_w,
ctx: ctx.clone(),
});
- let lose = driver.call_llm(RoutedRequest {
+ let lose = driver.call_model(RoutedRequest {
request,
decision: dec_l,
ctx,
@@ -1423,7 +1423,7 @@ mod tests {
) -> Result<Response> {
let offloads = futures::future::join_all((0..self.n).map(|i| {
let decision = test_decision(format!("m{i}"));
- driver.call_llm(RoutedRequest {
+ driver.call_model(RoutedRequest {
request: request.clone(),
decision,
ctx: ctx.clone(),
diff --git a/crates/libsy/src/core/testing.rs b/crates/libsy/src/core/testing.rs
index 226e55810..2643d4b94 100644
--- a/crates/libsy/src/core/testing.rs
+++ b/crates/libsy/src/core/testing.rs
@@ -20,7 +20,7 @@ use switchyard_protocol::{
Context, Decision, LlmClientError, LlmResponse, Request, Response, text_response,
};
-use crate::core::algorithm::{Algorithm, CallLlmRequest};
+use crate::core::algorithm::{Algorithm, CallModelRequest};
use crate::{LibsyError, Result};
/// The result a fake client hands back for one offloaded call.
@@ -58,7 +58,7 @@ pub(crate) async fn test_drive(
/// Serve one call and fulfill its promise, mapping failures the way a host does so
/// error-shape assertions match production.
-async fn fulfill(serve: Arc<impl Serve>, call: CallLlmRequest) -> Result<()> {
+async fn fulfill(serve: Arc<impl Serve>, call: CallModelRequest) -> Result<()> {
let routed = call.get_routed().clone();
let target = routed.decision.selected_model_id().to_string();
let result = serve
diff --git a/crates/libsy/src/lib.rs b/crates/libsy/src/lib.rs
index d40328136..1f2bf462f 100644
--- a/crates/libsy/src/lib.rs
+++ b/crates/libsy/src/lib.rs
@@ -6,7 +6,7 @@
mod core;
pub use core::algorithm::{
- Algorithm, CallLlmRequest, Driver, LlmTarget, LlmTargetSet, RoutedRequest, Step, StepStream,
+ Algorithm, CallModelRequest, Driver, LlmTarget, LlmTargetSet, RoutedRequest, Step, StepStream,
drive,
};
pub use core::classifier::{Classification, Classifier, Score};
diff --git a/crates/switchyard-server/tests/server.rs b/crates/switchyard-server/tests/server.rs
index 777830fdd..0e3272cb0 100644
--- a/crates/switchyard-server/tests/server.rs
+++ b/crates/switchyard-server/tests/server.rs
@@ -662,7 +662,7 @@ fn assert_in_order(haystack: &str, needles: &[&str]) {
/// A route is a synthetic model (`switchyard/classify`) with no upstream of its own; its
/// algorithm picks real targets, and *those* name the client. One request can emit several
-/// `Step::CallLlm` for different targets, and two targets may sit on different
+/// `Step::CallModel` for different targets, and two targets may sit on different
/// `[llm_clients.*]` sections — here the judge is on one provider and the serving models on
/// another. Pin that each call reaches its own target's upstream, rather than one client
/// chosen per route serving all of them.
diff --git a/docs/getting_started.md b/docs/getting_started.md
index 3dd0d039c..fc03dad73 100644
--- a/docs/getting_started.md
+++ b/docs/getting_started.md
@@ -263,7 +263,7 @@ behaviour.
### Drive the algorithm
-An algorithm yields a stream of steps. Each `Step::CallLlm` is a model call your
+An algorithm yields a stream of steps. Each `Step::CallModel` is a model call your
host performs over its own transport, and the run ends with
`Step::ReturnToAgent` carrying the final response. Serving those calls yourself
is what lets libsy embed in a host that already owns its HTTP stack, retries,