Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.
TypeScript agentaiai-agentsall-in-onecrmcrm-systememaillinearmcpmessagingnotesnotionofficerustslackslack-alternativestartupstartup-toolstypescriptworkspace
//! Model routing.
//!
//! Routing turns a `provider/model` api id into a runnable agent and owns the
//! provider fan-out so the rest of the crate stays provider-agnostic:
//!
//! - [`RoutedModel`] — the routed id bound to its provider client. One arm per
//! wire protocol: Anthropic-native, Gemini GenerateContent, OpenAI Responses,
//! and OpenAI-compatible Chat Completions. Compatible providers live in a
//! data registry keyed by name, so adding one is
//! `with_openai_provider` on the metered router.
//! - [`ProviderAgent`] — a built rig agent, with the same arms. Its
//! [`run_stream`](ProviderAgent::run_stream) matches internally, so callers
//! (e.g. `agent_loop`) hold one type and never fan out.
//!
//! Ids are addressed as `provider/model` (e.g. `anthropic/claude-opus-4-8`,
//! `fireworks/kimi-k3`); routing picks the provider from the segment, never by
//! sniffing the id. Unroutable ids fall back to the default model.
use std::collections::HashMap;
use std::sync::{Arc, OnceLock};
use std::time::{Duration, Instant};
use ai_toolset::RequestContext;
use ai_usage::{UsageContext, UsageRecorder};
use futures::StreamExt;
use macro_env_var::env_var;
use rig_agent::agent::{Agent, AgentBuilder, MultiTurnStreamItem};
use rig_agent::streaming::StreamingPrompt;
use rig_agent::tool::server::ToolServerHandle;
use rig_core::completion::{CompletionModel, GetTokenUsage};
use rig_core::http_client::{HttpClientExt, ReqwestClient};
use rig_core::message::Message;
use rig_core::providers::{anthropic, gemini, openai};
use rig_core::streaming::StreamedAssistantContent;
use tracing::Instrument as _;
use super::anthropic::{AnthropicModel, DeferredTools, LaidOutModel, SessionLayout};
use super::anthropic_prompt_layout::AnthropicPromptLayout;
use super::gemini::{GeminiCompletionModel, GeminiModel};
use super::metering::{MeteringContext, WireProtocol};
use super::metering_http::MeteredHttpClient;
use super::openai::{CompatibleCompletionModel, OpenAiChatCompletionsModel, OpenAiResponsesModel};
use super::types::Model;
use super::usage_amount::usage_amount;
use super::{ModelSpeed, PredefinedModel, ReasoningEffort};
use crate::agent_loop::{SystemPrompt, ToolSearch};
use crate::error::AgentError;
use crate::hook::{BridgeInputs, StreamBridge};
use crate::stream::{ChatCompletionStream, StreamPart};
use crate::telemetry::{ChatSpanHook, GenAiContext, TracedModel};
env_var! {
struct ApiKeys {
AnthropicApiKey,
OpenaiApiKey,
CerebrasApiKey,
/// Doppler name is `FIREWORK_API_KEY` (singular), from `shared_ai`.
FireworkApiKey,
GoogleGenerativeAiApiKey
}
}
/// Provider segment for native Anthropic.
const ANTHROPIC_PROVIDER: &str = "anthropic";
/// Provider segment the built-in OpenAI client is registered under.
const OPENAI_PROVIDER: &str = "openai";
/// Provider segment Cerebras is registered under (OpenAI-compatible Chat
/// Completions).
const CEREBRAS_PROVIDER: &str = "cerebras";
/// Cerebras inference endpoint (OpenAI-compatible Chat Completions API).
const CEREBRAS_BASE_URL: &str = "https://api.cerebras.ai/v1";
/// Provider segment Fireworks is registered under (OpenAI-compatible Chat
/// Completions). Open-weight models on the in-memory Macro agent route here.
const FIREWORKS_PROVIDER: &str = "fireworks";
/// Fireworks inference endpoint (OpenAI-compatible Chat Completions API).
const FIREWORKS_BASE_URL: &str = "https://api.fireworks.ai/inference/v1";
/// Provider segment Google Gemini is registered under (native GenerateContent).
const GOOGLE_PROVIDER: &str = "google";
/// Output cap for OpenAI-compatible Chat Completions hosts, whose limits
/// differ per host and model.
const COMPATIBLE_MAX_TOKENS: u64 = 16_000;
/// A routed model id bound to the provider client that serves it.
pub(crate) enum RoutedModel<'a, H = MeteredHttpClient> {
/// A model on Anthropic's native API.
Anthropic(AnthropicModel<'a, AnthropicPromptLayout<H>>),
/// A model on Gemini's native GenerateContent API.
Gemini(GeminiModel<'a, H>),
/// A model on the OpenAI-compatible Chat Completions API.
OpenAiChatCompletions(OpenAiChatCompletionsModel<'a, H>),
/// A model on OpenAI's Responses API.
OpenAiResponses(OpenAiResponsesModel<'a, H>),
}
impl<'a, H: HttpClientExt + Clone + Default + std::fmt::Debug + 'static> RoutedModel<'a, H> {
/// The provider segment of the routed id (`anthropic`, `openai`, …).
pub(crate) fn provider(&self) -> &str {
match self {
RoutedModel::Anthropic(m) => m.model().provider(),
RoutedModel::Gemini(m) => m.model().provider(),
RoutedModel::OpenAiChatCompletions(m) => m.model().provider(),
RoutedModel::OpenAiResponses(m) => m.model().provider(),
}
}
/// The wire protocol the provider speaks, which fixes its usage semantics.
pub(crate) fn protocol(&self) -> WireProtocol {
match self {
RoutedModel::Anthropic(_) => WireProtocol::Anthropic,
RoutedModel::Gemini(_) => WireProtocol::Gemini,
RoutedModel::OpenAiChatCompletions(_) => WireProtocol::ChatCompletions,
RoutedModel::OpenAiResponses(_) => WireProtocol::Responses,
}
}
/// The bare model name sent to the provider.
pub(crate) fn model_name(&self) -> &str {
match self {
RoutedModel::Anthropic(m) => m.model().name(),
RoutedModel::Gemini(m) => m.model().name(),
RoutedModel::OpenAiChatCompletions(m) => m.model().name(),
RoutedModel::OpenAiResponses(m) => m.model().name(),
}
}
/// Build the rig agent for this model, applying provider-specific thinking
/// config. Pure construction — no model call is made here.
///
/// Anthropic caches a split system prompt's shared part on its own, and a
/// model that loads tools by reference declares the whole catalog upfront.
/// Every other provider receives the prompt joined and loads tools by
/// registering them.
#[allow(clippy::too_many_arguments)]
pub(crate) fn into_agent(
self,
reasoning_effort: Option<ReasoningEffort>,
speed: ModelSpeed,
handle: ToolServerHandle,
system_prompt: &SystemPrompt,
tool_search: &ToolSearch,
max_turns: usize,
max_tokens: Option<u64>,
telemetry: &GenAiContext,
) -> ProviderAgent<H> {
let joined = system_prompt.joined();
// Unset means the model's maximum. Anthropic requires the field, so it
// gets the model's cap. The Responses and Gemini APIs default to the
// model's maximum when it's left out. OpenAI-compatible hosts default
// to their own, often small, limit, so they keep a fixed one.
let max_tokens = match &self {
RoutedModel::Anthropic(m) => Some(max_tokens.unwrap_or_else(|| m.max_output_tokens())),
RoutedModel::OpenAiChatCompletions(_) => {
Some(max_tokens.unwrap_or(COMPATIBLE_MAX_TOKENS))
}
RoutedModel::Gemini(_) | RoutedModel::OpenAiResponses(_) => max_tokens,
};
let system_prompt_text = joined.as_str();
match self {
RoutedModel::Anthropic(m) => {
let mut params = m
.thinking_params(reasoning_effort)
.unwrap_or_else(|| serde_json::json!({}));
speed.apply(&m.model().to_string(), &mut params);
let thinking = Some(params);
let layout = SessionLayout {
shared_system: system_prompt.shared().is_some(),
deferred: (m.loads_tools_by_reference() && !tool_search.catalog.is_empty())
.then(|| {
DeferredTools::new(&tool_search.catalog, tool_search.loads.clone())
}),
};
let (preamble, session_system) = match system_prompt.shared() {
Some(shared) => (shared, Some(system_prompt.rest())),
None => (system_prompt_text, None),
};
ProviderAgent::Anthropic(AnthropicAgent {
agent: build_agent(
LaidOutModel::new(m.completion(), layout),
thinking,
handle,
preamble,
max_turns,
max_tokens,
telemetry,
),
session_system,
})
}
RoutedModel::Gemini(m) => {
let thinking = m.thinking_params();
ProviderAgent::Gemini(build_agent(
m.completion(),
thinking,
handle,
system_prompt_text,
max_turns,
max_tokens,
telemetry,
))
}
RoutedModel::OpenAiChatCompletions(m) => {
let mut params = m
.thinking_params(reasoning_effort)
.unwrap_or_else(|| serde_json::json!({}));
speed.apply(&m.model().to_string(), &mut params);
let thinking = Some(params);
ProviderAgent::OpenAiChatCompletions(build_agent(
m.completion(),
thinking,
handle,
system_prompt_text,
max_turns,
max_tokens,
telemetry,
))
}
RoutedModel::OpenAiResponses(m) => {
let mut params = m
.thinking_params(reasoning_effort)
.unwrap_or_else(|| serde_json::json!({}));
speed.apply(&m.model().to_string(), &mut params);
let thinking = Some(params);
ProviderAgent::OpenAiResponses(build_agent(
m.completion(),
thinking,
handle,
system_prompt_text,
max_turns,
max_tokens,
telemetry,
))
}
}
}
}
/// A built rig agent bound to the provider serving the session's model.
///
/// The provider arms are different concrete `Agent<M>` types; [`run_stream`] hides
/// that behind one concrete [`ChatCompletionStream`], so callers never match.
///
/// [`run_stream`]: ProviderAgent::run_stream
pub(crate) enum ProviderAgent<
H: HttpClientExt + Clone + Default + std::fmt::Debug + 'static = MeteredHttpClient,
> {
/// An agent over Anthropic's native completion model.
Anthropic(AnthropicAgent<H>),
/// An agent over Gemini's native GenerateContent model.
Gemini(Agent<TracedModel<GeminiCompletionModel<H>>>),
/// An agent over the OpenAI Chat Completions model.
OpenAiChatCompletions(Agent<TracedModel<CompatibleCompletionModel<H>>>),
/// An agent over the OpenAI Responses model.
OpenAiResponses(Agent<TracedModel<openai::responses_api::ResponsesCompletionModel<H>>>),
/// A test-only agent over an arbitrary completion model (e.g. a scripted
/// fake), type-erased so arbitrary model types do not enter this enum.
#[cfg(test)]
Test(Box<dyn DynStreamAgent>),
}
impl<H: HttpClientExt + Clone + Default + std::fmt::Debug + 'static> ProviderAgent<H> {
/// Run the agentic loop and adapt rig's stream into the provider-agnostic
/// [`StreamPart`] stream consumed by DCS. The provider fan-out is internal.
#[allow(clippy::too_many_arguments)]
pub(crate) async fn run_stream(
&self,
prompt: Message,
history: Vec<Message>,
max_turns: usize,
inputs: BridgeInputs,
recorder: Arc<dyn UsageRecorder>,
usage_ctx: UsageContext,
model: String,
request_context: RequestContext,
telemetry: GenAiContext,
) -> ChatCompletionStream<'static> {
match self {
ProviderAgent::Anthropic(anthropic) => {
// After the agent's preamble, before the conversation: the
// second system block.
let history = anthropic
.session_system
.iter()
.map(Message::system)
.chain(history)
.collect();
drive_stream(
&anthropic.agent,
WireProtocol::Anthropic,
prompt,
history,
max_turns,
inputs,
recorder,
usage_ctx,
model,
request_context.clone(),
telemetry,
)
.await
}
ProviderAgent::Gemini(agent) => {
drive_stream(
agent,
WireProtocol::Gemini,
prompt,
history,
max_turns,
inputs,
recorder,
usage_ctx,
model,
request_context.clone(),
telemetry,
)
.await
}
ProviderAgent::OpenAiChatCompletions(agent) => {
drive_stream(
agent,
WireProtocol::ChatCompletions,
prompt,
history,
max_turns,
inputs,
recorder,
usage_ctx,
model,
request_context.clone(),
telemetry,
)
.await
}
ProviderAgent::OpenAiResponses(agent) => {
drive_stream(
agent,
WireProtocol::Responses,
prompt,
history,
max_turns,
inputs,
recorder,
usage_ctx,
model,
request_context.clone(),
telemetry,
)
.await
}
#[cfg(test)]
ProviderAgent::Test(agent) => {
agent
.run_stream_dyn(
prompt,
history,
max_turns,
inputs,
recorder,
usage_ctx,
model,
request_context.clone(),
telemetry,
)
.await
}
}
}
}
/// The Anthropic arm of [`ProviderAgent`]: the agent, whose preamble is the
/// system prompt's shared part when it is split, and the per-session rest.
pub(crate) struct AnthropicAgent<H: HttpClientExt + Clone + Default + std::fmt::Debug + 'static> {
agent: Agent<
TracedModel<LaidOutModel<anthropic::completion::CompletionModel<AnthropicPromptLayout<H>>>>,
>,
session_system: Option<String>,
}
/// Routes model api-id strings to the provider client that serves them.
///
/// Holds native Anthropic, Gemini, and OpenAI Responses clients plus a
/// registry of OpenAI-compatible Chat Completions clients keyed by provider
/// name. The built-in [`OPENAI_PROVIDER`] always uses Responses; register
/// compatible providers with `with_openai_provider` on the metered router.
#[derive(Clone)]
pub struct ModelRouter<H = ReqwestClient> {
anthropic: Arc<anthropic::Client<AnthropicPromptLayout<H>>>,
openai: Arc<openai::Client<H>>,
gemini: Option<Arc<gemini::Client<H>>>,
openai_compatible: HashMap<String, Arc<openai::CompletionsClient<H>>>,
}
impl<H: HttpClientExt + Clone + 'static> ModelRouter<H> {
/// Build a router over native Anthropic and OpenAI Responses clients, with
/// no OpenAI-compatible Chat Completions providers registered yet. The
/// Anthropic client sends through [`AnthropicPromptLayout`].
pub fn new(
anthropic: anthropic::Client<AnthropicPromptLayout<H>>,
openai: openai::Client<H>,
) -> Self {
Self {
anthropic: Arc::new(anthropic),
openai: Arc::new(openai),
gemini: None,
openai_compatible: HashMap::new(),
}
}
}
impl ModelRouter {
/// Build a router with the built-in providers from the environment.
///
/// Requires `ANTHROPIC_API_KEY`, `OPENAI_API_KEY`, `CEREBRAS_API_KEY`,
/// `FIREWORK_API_KEY`, and `GOOGLE_GENERATIVE_AI_API_KEY`. Chain
/// `with_openai_provider` on the returned router to add more.
pub fn try_from_env() -> Result<ModelRouter<MeteredHttpClient>, AgentError> {
let env = ApiKeys::new()?;
// No executions may hide below the financial HTTP boundary.
let http = ReqwestClient::builder()
.retry(reqwest::retry::never())
.redirect(reqwest::redirect::Policy::none())
.build()
.map_err(|error| rig_core::http_client::Error::Instance(Box::new(error)))?;
let anthropic = anthropic::Client::builder()
.api_key(env.anthropic_api_key.to_string())
.http_client(AnthropicPromptLayout::new(MeteredHttpClient::new(
http.clone(),
ANTHROPIC_PROVIDER,
WireProtocol::Anthropic,
)))
.build()?;
// Default base URL is api.openai.com; OpenAI's GPT models use
// Responses API so reasoning models get max_output_tokens.
let openai = openai::Client::builder()
.api_key(env.openai_api_key.to_string())
.http_client(MeteredHttpClient::new(
http.clone(),
OPENAI_PROVIDER,
WireProtocol::Responses,
))
.build()?;
// Cerebras and Fireworks speak the OpenAI Chat Completions API, so they
// ride the compatible-provider registry: a `<provider>/` segment routes
// to the client registered under that name. Gemini uses GenerateContent
// so tool-call thought signatures survive the multi-step tool loop.
let gemini = gemini::Client::builder()
.api_key(env.google_generative_ai_api_key.to_string())
.http_client(MeteredHttpClient::new(
http.clone(),
GOOGLE_PROVIDER,
WireProtocol::Gemini,
))
.build()?;
ModelRouter::new(anthropic, openai)
.with_gemini_client(gemini)
.with_openai_provider(CEREBRAS_PROVIDER, CEREBRAS_BASE_URL, &env.cerebras_api_key)?
.with_openai_provider(
FIREWORKS_PROVIDER,
FIREWORKS_BASE_URL,
&env.firework_api_key,
)
}
/// The process-wide full router, built from the environment on first use.
///
/// This is the only router the crate uses — every entry point routes through
/// the same fully-populated instance, so a model id resolves identically
/// everywhere. Register additional OpenAI-compatible providers here as they
/// are added.
pub(crate) fn shared() -> Result<&'static ModelRouter<MeteredHttpClient>, AgentError> {
static ROUTER: OnceLock<ModelRouter<MeteredHttpClient>> = OnceLock::new();
if let Some(router) = ROUTER.get() {
return Ok(router);
}
let router = Self::try_from_env()?;
Ok(ROUTER.get_or_init(|| router))
}
}
impl<H: HttpClientExt + Clone + 'static> ModelRouter<H> {
/// Bind the native Gemini GenerateContent client. Google ids route here.
pub fn with_gemini_client(mut self, client: gemini::Client<H>) -> Self {
self.gemini = Some(Arc::new(client));
self
}
/// Register an already-built OpenAI-compatible Chat Completions client under
/// `provider`.
pub fn with_openai_client(
mut self,
provider: impl Into<String>,
client: openai::CompletionsClient<H>,
) -> Self {
self.openai_compatible
.insert(provider.into(), Arc::new(client));
self
}
}
impl ModelRouter<MeteredHttpClient> {
/// This router with `provider`'s client authenticating with `api_key`
/// instead of the deployment's key, for a turn an agent's own key pays for.
///
/// Only the native Anthropic and OpenAI clients can be rekeyed; any other
/// provider is [`AgentError::UnknownModel`]. The rebuilt client keeps its
/// own plain copy of the key as a header value, so drop the router with
/// the turn that needed it.
pub fn with_provider_api_key(&self, provider: &str, api_key: &str) -> Result<Self, AgentError> {
let http = ReqwestClient::builder()
.retry(reqwest::retry::never())
.redirect(reqwest::redirect::Policy::none())
.build()
.map_err(|error| rig_core::http_client::Error::Instance(Box::new(error)))?;
let mut router = self.clone();
match provider {
ANTHROPIC_PROVIDER => {
router.anthropic = Arc::new(
anthropic::Client::builder()
.api_key(api_key)
.http_client(AnthropicPromptLayout::new(MeteredHttpClient::new(
http,
ANTHROPIC_PROVIDER,
WireProtocol::Anthropic,
)))
.build()?,
);
}
OPENAI_PROVIDER => {
router.openai = Arc::new(
openai::Client::builder()
.api_key(api_key)
.http_client(MeteredHttpClient::new(
http,
OPENAI_PROVIDER,
WireProtocol::Responses,
))
.build()?,
);
}
other => return Err(AgentError::UnknownModel(other.to_owned())),
}
Ok(router)
}
/// Register an OpenAI-compatible Chat Completions provider from a base URL
/// and key.
///
/// This is the whole cost of adding a provider — models served by it are
/// then reachable as `provider/<model-id>`. The extension point for the
/// open provider set (Cerebras is wired this way in [`try_from_env`]).
///
/// [`try_from_env`]: ModelRouter::try_from_env
pub fn with_openai_provider(
self,
provider: impl Into<String>,
base_url: &str,
api_key: &str,
) -> Result<Self, AgentError> {
let provider = provider.into();
let http = ReqwestClient::builder()
.retry(reqwest::retry::never())
.redirect(reqwest::redirect::Policy::none())
.build()
.map_err(|error| rig_core::http_client::Error::Instance(Box::new(error)))?;
let client = openai::CompletionsClient::builder()
.api_key(api_key)
.base_url(base_url)
.http_client(MeteredHttpClient::new(
http,
provider.clone(),
WireProtocol::ChatCompletions,
))
.build()?;
Ok(self.with_openai_client(provider, client))
}
}
impl<H: HttpClientExt + Clone + Default + std::fmt::Debug + 'static> ModelRouter<H> {
/// Route a `provider/model` id to the provider that serves it.
///
/// Returns [`AgentError::UnknownModel`] if no provider claims it (and
/// [`AgentError::MalformedModel`] if the id has no `provider/` segment).
pub(crate) fn route<'a>(&self, model: &'a str) -> Result<RoutedModel<'a, H>, AgentError> {
let parsed = Model::try_from(model)?;
if parsed.provider() == ANTHROPIC_PROVIDER {
return Ok(RoutedModel::Anthropic(AnthropicModel::new(
parsed,
self.anthropic.clone(),
)));
}
if parsed.provider() == OPENAI_PROVIDER {
return Ok(RoutedModel::OpenAiResponses(OpenAiResponsesModel::new(
parsed,
self.openai.clone(),
)));
}
if parsed.provider() == GOOGLE_PROVIDER {
let Some(client) = &self.gemini else {
return Err(AgentError::UnknownModel(model.to_string()));
};
return Ok(RoutedModel::Gemini(GeminiModel::new(
parsed,
Arc::clone(client),
)));
}
if let Some(client) = self.openai_compatible.get(parsed.provider()) {
let client = Arc::clone(client);
return Ok(RoutedModel::OpenAiChatCompletions(
OpenAiChatCompletionsModel::new(parsed, client),
));
}
Err(AgentError::UnknownModel(model.to_string()))
}
/// Route `model`, falling back to the default model on an unroutable id.
pub(crate) fn route_or_default<'a>(&self, model: &'a str) -> RoutedModel<'a, H> {
self.route(model).unwrap_or_else(|_| self.default_model())
}
/// The fallback model: native Anthropic serving [`PredefinedModel::Smart`].
///
/// Built via `From<PredefinedModel>` so the bound [`Model`] carries the
/// bare api id — `PredefinedModel`'s `Display` is the provider-qualified
/// routing id, which the Anthropic API rejects as a model name.
fn default_model(&self) -> RoutedModel<'static, H> {
RoutedModel::Anthropic(AnthropicModel::new(
PredefinedModel::Smart.into(),
self.anthropic.clone(),
))
}
}
/// Build a rig agent from a completion model and per-session config.
///
/// The model is wrapped in [`TracedModel`] so every model call records its
/// request on the `chat` span. rig's own content recording stays off — it is
/// unbounded; `crate::telemetry` records bounded content instead.
fn build_agent<M: CompletionModel>(
model: M,
thinking: Option<serde_json::Value>,
handle: ToolServerHandle,
system_prompt: &str,
max_turns: usize,
max_tokens: Option<u64>,
telemetry: &GenAiContext,
) -> Agent<TracedModel<M>> {
let mut builder = AgentBuilder::new(TracedModel::new(model, telemetry.clone()))
.name(telemetry.agent_name())
.record_content_telemetry(false)
.tool_server_handle(handle)
.default_max_turns(max_turns)
.preamble(system_prompt);
if let Some(max_tokens) = max_tokens {
builder = builder.max_tokens(max_tokens);
}
if let Some(params) = thinking {
builder = builder.additional_params(params);
}
builder.build()
}
/// Run the agentic loop on `agent` and adapt rig's stream into the
/// provider-agnostic [`StreamPart`] stream consumed by DCS.
#[allow(clippy::too_many_arguments)]
async fn drive_stream<M>(
agent: &Agent<M>,
protocol: WireProtocol,
prompt: Message,
history: Vec<Message>,
max_turns: usize,
inputs: BridgeInputs,
recorder: Arc<dyn UsageRecorder>,
usage_ctx: UsageContext,
model: String,
request_context: RequestContext,
telemetry: GenAiContext,
) -> ChatCompletionStream<'static>
where
M: CompletionModel + 'static,
M::StreamingResponse: GetTokenUsage + Send + Sync,
{
// The caller's `invoke_agent` span (see `Session::send_message`): rig adopts
// it as the run's agent span but never records onto a span it did not
// open, so the run's input, output and usage are recorded here.
let agent_span = tracing::Span::current();
telemetry.record_agent_input(&agent_span, &prompt);
let (bridge, mut rx) = StreamBridge::channel(
inputs,
request_context.searchable_tools.clone(),
request_context.cancel.clone(),
);
// Driver-side sender for parts derived from rig stream items (thinking,
// usage, errors). The lifecycle hooks (text, tool call, tool response) send
// through their own clone inside `bridge`; both feed the same FIFO channel.
let driver_tx = bridge.sender();
let mut rig_stream = agent
.stream_prompt(prompt)
.history(history)
.max_turns(max_turns)
.max_invalid_tool_call_retries(crate::hook::RIG_INVALID_TOOL_CALL_RETRIES)
.add_hook(bridge)
.add_hook(ChatSpanHook(telemetry.clone()))
.await;
// Drive the rig stream on its own task. The hook emits a tool call the
// moment the model finishes it — *before* the (often slow) tool executes —
// but rig runs that execution inside a single `rig_stream.next()` poll, so
// draining the channel only between polls would hold the pending tool call
// hidden until its response landed. Polling the rig stream here, off the
// consumer's path, lets every hook-emitted part flow through `rx` and out
// to the client as soon as it is produced — so a tool call renders in its
// pending state immediately and its response renders when execution
// finishes.
// Whatever ends the driver - the stream running dry, a provider error, an
// abort when the consumer drops the stream - the model call's parked
// `chat` span is released with it (see `GenAiContext::finish_run`), and a
// run that never reached its final response or an error is recorded as
// cancelled: the consumer went away, and the run went with it.
struct FinishRun {
telemetry: GenAiContext,
agent_span: tracing::Span,
concluded: bool,
}
impl Drop for FinishRun {
fn drop(&mut self) {
if !self.concluded {
self.telemetry.record_agent_failure(
&self.agent_span,
"cancelled",
genai_telemetry::attr::finish_reason::CANCELLED,
"the run was cancelled before the agent answered",
);
}
self.telemetry.finish_run();
}
}
/// Liveness of the provider stream, recorded on the run's span by `Drop` so
/// it lands however the driver ends - including the abort a cancelled turn
/// causes, which is the case most worth seeing.
///
/// These answer what a parked stream cannot: the task is idle whether the
/// provider is dribbling tokens or has gone silent, and only the timing of
/// the items tells those apart. Read `trailing_silence_ms` first - near
/// zero means the model was still producing when the run ended, a long tail
/// means it had stopped talking to us well before.
///
/// `first_chunk_ms` is the run's first model call's time to first chunk,
/// measured from the provider request (the same value as that call's
/// `chat` span's `macro.genai.chat.time_to_first_chunk_ms`), so the
/// provider's latency reads off the run span without a join.
struct StreamLiveness {
span: tracing::Span,
telemetry: GenAiContext,
started_at: Instant,
items: i64,
last_item: Option<Duration>,
}
impl StreamLiveness {
fn new(span: tracing::Span, telemetry: GenAiContext) -> Self {
telemetry.take_run_first_chunk();
Self {
span,
telemetry,
started_at: Instant::now(),
items: 0,
last_item: None,
}
}
fn observed(&mut self) {
self.items += 1;
self.last_item = Some(self.started_at.elapsed());
}
}
impl Drop for StreamLiveness {
// Recorded as `i64`: a `u64` reaches OpenTelemetry as a *string*
// attribute, which every numeric query would then silently miss.
fn drop(&mut self) {
self.span.record("agent.stream.items", self.items);
if let Some((time_to_first_chunk, kind)) = self.telemetry.take_run_first_chunk() {
self.span
.record("agent.stream.first_chunk_ms", millis(time_to_first_chunk));
self.span.record("agent.stream.first_chunk_kind", kind);
}
// With nothing ever received the silence is the whole run, which is
// what `unwrap_or_default` says here.
let silent_since = self.last_item.unwrap_or_default();
self.span.record(
"agent.stream.trailing_silence_ms",
millis(self.started_at.elapsed().saturating_sub(silent_since)),
);
}
}
/// Whole milliseconds, saturating: a duration longer than `i64::MAX`
/// milliseconds is not a number anybody needs exactly.
fn millis(duration: Duration) -> i64 {
i64::try_from(duration.as_millis()).unwrap_or(i64::MAX)
}
let mut finish_run = FinishRun {
telemetry: telemetry.clone(),
agent_span: agent_span.clone(),
concluded: false,
};
let driver_span = agent_span.clone();
let financial_context = MeteringContext::current();
let records_per_call = financial_context
.as_ref()
.is_some_and(MeteringContext::records_per_call);
let driver = tokio::spawn(
MeteringContext::carry(financial_context, async move {
let mut liveness = StreamLiveness::new(agent_span.clone(), telemetry.clone());
while let Some(item) = rig_stream.next().await {
liveness.observed();
if let Ok(MultiTurnStreamItem::StreamAssistantItem(content)) = &item
&& let Some(kind) = chunk_kind(content)
{
telemetry.record_chunk(kind);
}
match item {
Ok(MultiTurnStreamItem::StreamAssistantItem(
StreamedAssistantContent::ReasoningDelta { reasoning, .. },
)) => {
// Forwarded as it arrives, like a text delta. Held back
// until the model says something else, a turn that
// thinks for half a minute before its first tool call
// shows the reader nothing for that whole time and then
// the entire thought at once.
let _ = driver_tx.send(Ok(StreamPart::Thinking(reasoning)));
}
other => {
match other {
Ok(MultiTurnStreamItem::FinalResponse(final_resp)) => {
let usage = final_resp.usage;
finish_run.concluded = true;
telemetry.record_agent_output(
&agent_span,
&final_resp.output,
&usage,
);
// Aggregate analytics only. Financial evidence is
// persisted per HTTP attempt before SDK parsing.
if !records_per_call {
recorder.record(
usage_ctx.clone().into_event(
model.clone(),
usage_amount(protocol, &usage),
),
);
}
let _ =
driver_tx.send(Ok(StreamPart::Usage(crate::stream::Usage {
input_tokens: usage.input_tokens,
output_tokens: usage.output_tokens,
})));
}
Err(e) => {
finish_run.concluded = true;
let error = AgentError::Streaming(e);
if error.was_cancelled() {
// The caller stopped the run through its
// cancellation token: a stop, not a fault.
telemetry.record_agent_failure(
&agent_span,
"cancelled",
genai_telemetry::attr::finish_reason::CANCELLED,
"the run was cancelled",
);
} else {
// A provider error, or the runtime giving
// up (retries exhausted): the run failed.
telemetry.record_agent_failure(
&agent_span,
"streaming_error",
genai_telemetry::attr::finish_reason::ERROR,
&error.to_string(),
);
}
let _ = driver_tx.send(Err(error));
}
_ => {}
}
}
}
}
// Dropping `rig_stream` (and with it the hook's sender) plus `driver_tx`
// here closes the channel, ending the consumer stream below.
drop(finish_run);
})
// Entered into the agent span: the runtime opens its `chat` and
// `execute_tool` spans from inside this task, and they belong under
// the run, not at the root of a trace of their own.
.instrument(driver_span),
);
// Abort the driver when the consumer drops the returned stream (e.g. on
// cancellation), which drops `rig_stream` and cancels any in-flight tool —
// matching the prior behaviour where the rig stream lived inline.
struct AbortOnDrop(tokio::task::JoinHandle<()>);
impl Drop for AbortOnDrop {
fn drop(&mut self) {
self.0.abort();
}
}
let guard = AbortOnDrop(driver);
let stream = async_stream::stream! {
let _guard = guard;
while let Some(part) = rx.recv().await {
yield part;
}
};
Box::pin(stream)
}
/// What a streamed item carries, for the model call's first-chunk timing.
/// `None` for the final response and items rig does not model.
fn chunk_kind<R>(content: &StreamedAssistantContent<R>) -> Option<&'static str> {
match content {
StreamedAssistantContent::Text(_) => Some("text"),
StreamedAssistantContent::Reasoning(_)
| StreamedAssistantContent::ReasoningDelta { .. } => Some("reasoning"),
StreamedAssistantContent::ToolCall { .. }
| StreamedAssistantContent::ToolCallDelta { .. } => Some("tool_call"),
StreamedAssistantContent::Final(_) | StreamedAssistantContent::Unknown(_) => None,
}
}
/// Test-only type erasure so [`ProviderAgent`] can hold an arbitrary
/// [`Agent<M>`] (e.g. a scripted fake model) without adding a model type parameter.
/// Mirrors the production arms: it just drives [`drive_stream`].
#[cfg(test)]
pub(crate) trait DynStreamAgent: Send + Sync {
#[allow(clippy::too_many_arguments)]
fn run_stream_dyn<'a>(
&'a self,
prompt: Message,
history: Vec<Message>,
max_turns: usize,
inputs: BridgeInputs,
recorder: Arc<dyn UsageRecorder>,
usage_ctx: UsageContext,
model: String,
request_context: RequestContext,
telemetry: GenAiContext,
) -> std::pin::Pin<
Box<dyn std::future::Future<Output = ChatCompletionStream<'static>> + Send + 'a>,
>;
}
#[cfg(test)]
impl<M> DynStreamAgent for Agent<M>
where
M: CompletionModel + 'static,
M::StreamingResponse: GetTokenUsage + Send + Sync,
{
fn run_stream_dyn<'a>(
&'a self,
prompt: Message,
history: Vec<Message>,
max_turns: usize,
inputs: BridgeInputs,
recorder: Arc<dyn UsageRecorder>,
usage_ctx: UsageContext,
model: String,
request_context: RequestContext,
telemetry: GenAiContext,
) -> std::pin::Pin<
Box<dyn std::future::Future<Output = ChatCompletionStream<'static>> + Send + 'a>,
> {
// Scripted fakes report Anthropic-shaped usage: input excludes cache.
Box::pin(drive_stream(
self,
WireProtocol::Anthropic,
prompt,
history,
max_turns,
inputs,
recorder,
usage_ctx,
model,
request_context,
telemetry,
))
}
}
/// A routed agent over a test transport runs exactly as production's does.
#[cfg(test)]
impl<H> DynStreamAgent for ProviderAgent<H>
where
H: HttpClientExt + Clone + Default + std::fmt::Debug + Send + Sync + 'static,
{
fn run_stream_dyn<'a>(
&'a self,
prompt: Message,
history: Vec<Message>,
max_turns: usize,
inputs: BridgeInputs,
recorder: Arc<dyn UsageRecorder>,
usage_ctx: UsageContext,
model: String,
request_context: RequestContext,
telemetry: GenAiContext,
) -> std::pin::Pin<
Box<dyn std::future::Future<Output = ChatCompletionStream<'static>> + Send + 'a>,
> {
Box::pin(self.run_stream(
prompt,
history,
max_turns,
inputs,
recorder,
usage_ctx,
model,
request_context,
telemetry,
))
}
}
#[cfg(test)]
impl ProviderAgent {
/// Build a test-only [`ProviderAgent`] backed by `model` (a fake completion
/// model), wired through the same [`build_agent`] used in production.
pub(crate) fn test<M>(
model: M,
system_prompt: &str,
max_turns: usize,
max_tokens: Option<u64>,
handle: ToolServerHandle,
telemetry: GenAiContext,
) -> Self
where
M: CompletionModel + 'static,
M::StreamingResponse: GetTokenUsage + Send + Sync,
{
telemetry.set_model("test", "fake-model");
ProviderAgent::Test(Box::new(build_agent(
model,
None,
handle,
system_prompt,
max_turns,
max_tokens,
&telemetry,
)))
}
}