Make Codex and Claude Code subscriptions go further: a local model router that gives routine agent work to economical models and keeps your strongest model for the hard parts.
Rust ai-agentsai-gatewayclaude-codecodexcoding-agentcost-optimizationdeepseekllm-gatewayllm-routerlocal-firstmodel-routingrusttask-routingtauri
use bytes::Bytes;
use std::sync::Arc;
use crate::content_ref::ContentRef;
use crate::server::core_runtime::model_ir::{ModelEvent, ModelStreamEventV1, ModelUsage};
use hiroute_gateway_core::transport::GatewayResponseHead;
use super::contracts::CONVERSATION_CONTENT_PORT_DIGEST;
use super::crypto::{WorkspaceHmac, stable_id};
use super::otel::emit_content_ref;
use super::request::AttemptObservation;
use super::schema::{CONVERSATION_CONTENT_SCHEMA, ContentRefV1, ConversationContentEnvelopeV1};
use super::{RequestObservation, unix_nanos};
use hiroute_diagnostics::event::{
CaptureAbortReason, CaptureOutcome, ContentCaptureEnd, ContentRole, DiagnosticEvent,
};
const CONTENT_CHUNK_BYTES: usize = 64 * 1024;
const CANONICALIZATION_VERSION: &str = "hiroute.model-request-ir/v1";
const CANONICAL_RESPONSE_EVENT_MEDIA_TYPE: &str =
"application/vnd.hiroute.model-stream-event+json;version=1";
#[path = "content/request.rs"]
mod request;
#[derive(Default)]
pub(super) struct CanonicalFrameDelivery {
pub(super) events: Vec<ModelStreamEventV1>,
pub(super) usage: Option<ModelUsage>,
// Keep correlation storage charged until its accepted content is published.
pub(super) reservations: Vec<hiroute_gateway_core::runtime::body::Reservation>,
}
impl CanonicalFrameDelivery {
pub(super) fn merge_usage(&mut self, usage: &ModelUsage) {
self.usage
.get_or_insert_with(ModelUsage::default)
.merge_from(usage);
}
pub(super) fn merge(&mut self, mut other: Self) {
self.events.append(&mut other.events);
self.reservations.append(&mut other.reservations);
if let Some(usage) = other.usage {
self.merge_usage(&usage);
}
}
}
impl RequestObservation {
pub(super) fn begin_content(
&self,
direction: &'static str,
attempt: Option<&AttemptObservation>,
) {
if !self.captures_content() {
return;
}
let (should_emit, fork_id, parent_transcript_root) = {
let mut state = self.lock_state();
let started = match direction {
"request_input" => &mut state.request_content_started,
_ => &mut state.response_content_started,
};
if *started {
return;
}
*started = true;
let fork_id = self.fork_id(direction, attempt);
let parent_transcript_root = (direction == "response_delivered")
.then(|| state.request_result_transcript_root.clone())
.flatten();
let mut transcript = WorkspaceHmac::new(&self.inner.key, b"transcript-root");
transcript.update(CANONICALIZATION_VERSION.as_bytes());
transcript.update(
parent_transcript_root
.as_deref()
.unwrap_or("no-parent-root")
.as_bytes(),
);
transcript.update(b"append");
transcript.update(self.metadata().request_id.as_bytes());
transcript.update(direction.as_bytes());
transcript.update(fork_id.as_bytes());
match direction {
"request_input" => state.request_transcript = Some(transcript),
_ => state.response_transcript = Some(transcript),
}
(true, fork_id, parent_transcript_root)
};
if should_emit {
self.emit_content(ContentEvent {
direction,
phase: "begin",
attempt_id: attempt.map(|value| value.attempt_id.clone()),
fork_id,
parent_transcript_root,
result_transcript_root: None,
message_instance_id: None,
message_role: None,
content_kind: None,
content_id: None,
content_blob_digest: None,
message_ordinal: None,
part_ordinal: None,
chunk_ordinal: None,
transport_frame_id: None,
canonical_media_type: None,
canonical_bytes_base64: None,
content_ref: None,
downstream_delivery: None,
abort_reason: None,
completeness_delta: None,
});
}
}
fn append_content_chunk(&self, chunk: ContentChunk<'_>) {
if !self.captures_content() {
return;
}
let message_instance_id =
self.message_instance_id(chunk.direction, chunk.message_ordinal, chunk.role);
let content_id =
self.content_id(&message_instance_id, chunk.part_ordinal, chunk.whole_digest);
let parent_transcript_root = self.content_parent_root(chunk.direction);
let (chunk_ordinal, attempt_id, fork_id) = {
let mut state = self.lock_state();
let current = if chunk.direction == "request_input" {
if state.request_content_terminal {
return;
}
let current = state.request_content_ordinal;
state.request_content_ordinal = current.saturating_add(1);
if chunk.starts_content
&& let Some(transcript) = &mut state.request_transcript
{
update_transcript(transcript, &message_instance_id, &content_id);
}
current
} else {
if state.response_content_terminal {
return;
}
let current = state.response_content_ordinal;
state.response_content_ordinal = current.saturating_add(1);
if chunk.starts_content
&& let Some(transcript) = &mut state.response_transcript
{
update_transcript(transcript, &message_instance_id, &content_id);
}
current
};
let stats = if chunk.direction == "request_input" {
&mut state.request_content_stats
} else {
&mut state.response_content_stats
};
stats.note_chunk(chunk.bytes.len());
(
current,
chunk.attempt.map(|attempt| attempt.attempt_id.clone()),
self.fork_id(chunk.direction, chunk.attempt),
)
};
self.emit_content(ContentEvent {
direction: chunk.direction,
phase: "append",
attempt_id,
fork_id,
parent_transcript_root,
result_transcript_root: None,
message_instance_id: Some(message_instance_id),
message_role: Some(chunk.role.into()),
content_kind: Some(chunk.kind.into()),
content_id: Some(content_id),
content_blob_digest: Some(chunk.whole_digest.into()),
message_ordinal: Some(chunk.message_ordinal),
part_ordinal: Some(chunk.part_ordinal),
chunk_ordinal: Some(chunk_ordinal),
transport_frame_id: chunk.transport_frame_id.map(Into::into),
canonical_media_type: Some(chunk.media_type.into()),
canonical_bytes_base64: Some(base64_encode(chunk.bytes)),
content_ref: chunk.public_ref,
downstream_delivery: chunk.downstream_delivery.map(Into::into),
abort_reason: None,
completeness_delta: None,
});
}
pub(super) fn terminal_content(
&self,
direction: &'static str,
phase: &'static str,
abort_reason: Option<&str>,
) {
if !self.captures_content() {
return;
}
let (started, already_terminal, transcript, attempt) = {
let mut state = self.lock_state();
match direction {
"request_input" => {
let started = state.request_content_started;
let terminal = state.request_content_terminal;
if started && !terminal {
state.request_content_terminal = true;
}
(started, terminal, state.request_transcript.take(), None)
}
_ => {
let started = state.response_content_started;
let terminal = state.response_content_terminal;
if started && !terminal {
state.response_content_terminal = true;
}
(
started,
terminal,
state.response_transcript.take(),
state.accepted_attempt.clone(),
)
}
}
};
if !started || already_terminal {
return;
}
let stats = self.take_content_stats(direction);
self.emit_diagnostic(DiagnosticEvent::ContentCaptureEnd(ContentCaptureEnd {
role: match direction {
"request_input" => ContentRole::Request,
_ => ContentRole::Response,
},
outcome: if phase == "finish" {
CaptureOutcome::Success
} else {
CaptureOutcome::Abort
},
bytes: stats.bytes,
chunks: stats.chunks,
read_calls: stats.read_calls,
short_reads: stats.short_reads,
max_chunk_bytes: stats.max_chunk_bytes,
abort: abort_reason.map(capture_abort_reason),
}));
let result_transcript_root = transcript.map(|transcript| {
format!(
"transcript-{}",
transcript.finish().trim_start_matches("sha256:")
)
});
if direction == "request_input" {
self.lock_state().request_result_transcript_root = result_transcript_root.clone();
}
self.emit_content(ContentEvent {
direction,
phase,
attempt_id: attempt.as_ref().map(|value| value.attempt_id.clone()),
fork_id: self.fork_id(direction, attempt.as_ref()),
parent_transcript_root: self.content_parent_root(direction),
result_transcript_root,
message_instance_id: None,
message_role: None,
content_kind: None,
content_id: None,
content_blob_digest: None,
message_ordinal: None,
part_ordinal: None,
chunk_ordinal: None,
transport_frame_id: None,
canonical_media_type: None,
canonical_bytes_base64: None,
content_ref: None,
downstream_delivery: (direction == "response_delivered")
.then(|| "full_frame_transport_accepted".into()),
abort_reason: abort_reason.map(Into::into),
completeness_delta: Some(if phase == "finish" {
"complete".into()
} else {
"partial".into()
}),
});
}
fn content_blob_digest(&self, media_type: &str, bytes: &[u8]) -> String {
let mut digest = WorkspaceHmac::new(&self.inner.key, b"conversation-content-blob");
digest.update(CANONICALIZATION_VERSION.as_bytes());
digest.update(media_type.as_bytes());
digest.update_stream(bytes);
format!("blob-{}", digest.finish().trim_start_matches("sha256:"))
}
fn message_instance_id(&self, direction: &str, message_ordinal: u32, role: &str) -> String {
let parent_transcript_root = self.content_parent_root(direction);
stable_id(
"message",
&self.inner.key,
b"conversation-message-instance",
&[
self.metadata().conversation_id.as_bytes(),
parent_transcript_root
.as_deref()
.unwrap_or("no-parent-root")
.as_bytes(),
self.metadata().request_id.as_bytes(),
direction.as_bytes(),
&message_ordinal.to_be_bytes(),
role.as_bytes(),
],
)
}
fn content_parent_root(&self, direction: &str) -> Option<String> {
(direction == "response_delivered")
.then(|| self.lock_state().request_result_transcript_root.clone())
.flatten()
}
fn content_id(
&self,
message_instance_id: &str,
part_ordinal: u32,
content_blob_digest: &str,
) -> String {
stable_id(
"content",
&self.inner.key,
b"conversation-content-id",
&[
message_instance_id.as_bytes(),
&part_ordinal.to_be_bytes(),
content_blob_digest.as_bytes(),
],
)
}
fn public_content_ref(
&self,
reference: &ContentRef,
media_type: &str,
content_id: String,
digest: String,
) -> ContentRefV1 {
ContentRefV1 {
content_id,
digest,
byte_count: reference.byte_len(),
media_type: media_type.into(),
}
}
fn fork_id(&self, direction: &str, attempt: Option<&AttemptObservation>) -> String {
stable_id(
"fork",
&self.inner.key,
b"conversation-fork",
&[
self.metadata().request_id.as_bytes(),
direction.as_bytes(),
attempt.map_or(b"request".as_slice(), |value| value.attempt_id.as_bytes()),
],
)
}
fn emit_content(&self, event: ContentEvent) {
let correlation = self.correlation();
let request_id = self.metadata().request_id.clone();
let key = self.inner.key;
let event_kind = event.phase;
self.inner
.channels
.content
.publish(move |producer, sequence, loss| {
let sequence_bytes = sequence.to_be_bytes();
ConversationContentEnvelopeV1 {
schema_version: CONVERSATION_CONTENT_SCHEMA.into(),
schema_digest: CONVERSATION_CONTENT_PORT_DIGEST.into(),
channel: "conversation_content".into(),
producer,
sequence,
event_id: stable_id(
"content-event",
&key,
b"conversation-content-event",
&[
request_id.as_bytes(),
event.direction.as_bytes(),
event_kind.as_bytes(),
&sequence_bytes,
],
),
correlation,
direction: event.direction.into(),
phase: event.phase.into(),
attempt_id: event.attempt_id,
fork_id: event.fork_id,
parent_transcript_root: event.parent_transcript_root,
result_transcript_root: event.result_transcript_root,
message_instance_id: event.message_instance_id,
message_role: event.message_role,
content_kind: event.content_kind,
content_id: event.content_id,
content_blob_digest: event.content_blob_digest,
message_ordinal: event.message_ordinal,
part_ordinal: event.part_ordinal,
chunk_ordinal: event.chunk_ordinal,
transport_frame_id: event.transport_frame_id,
canonical_media_type: event.canonical_media_type,
canonical_bytes_base64: event.canonical_bytes_base64,
content_ref: event.content_ref,
downstream_delivery: event.downstream_delivery,
abort_reason: event.abort_reason,
occurred_at_unix_nanos: unix_nanos(),
completeness_delta: loss
.as_ref()
.map(|_| "partial".into())
.or(event.completeness_delta),
loss_watermark: loss,
}
});
}
}
pub struct AcceptedResponseCapture {
request: RequestObservation,
attempt: Option<Arc<AttemptObservation>>,
ended: bool,
}
impl AcceptedResponseCapture {
pub fn new(request: RequestObservation) -> Self {
Self {
request,
attempt: None,
ended: false,
}
}
pub fn accepted_head(&mut self, _head: &GatewayResponseHead) {}
/// Must be called only after the inner transport accepted the complete
/// body frame. The caller retains ownership of this sequencing rule.
pub fn accepted_frame(&mut self, body: &Bytes, end_stream: bool) {
if !self.request.is_enabled() {
if self.request.tracks_attempts() && (!body.is_empty() || end_stream) {
// Diagnostic acceptance has no content identity or capture.
// Business channels remain disabled by RequestObservation.
let _ = self.request.accept_current("", body.len());
}
return;
}
if !body.is_empty() || end_stream {
let frame_ordinal = self.request.lock_state().response_frame_ordinal;
let frame_id = stable_id(
"frame",
&self.request.inner.key,
b"downstream-accepted-frame",
&[
self.request.metadata().request_id.as_bytes(),
&frame_ordinal.to_be_bytes(),
body,
],
);
if let Some(attempt) = self.request.accept_current(&frame_id, body.len()) {
let capture_attempt = self
.attempt
.get_or_insert_with(|| Arc::new(attempt.clone()))
.clone();
let result = self
.request
.response_capture()
.ok_or("canonical_capture_not_bound")
.and_then(|capture| {
capture.accepted_frame(
self.request.clone(),
capture_attempt,
frame_id,
body,
end_stream,
)
});
if let Err(reason) = result {
self.request
.begin_content("response_delivered", Some(&attempt));
self.request
.terminal_content("response_delivered", "abort", Some(reason));
}
}
}
if end_stream {
self.ended = true;
self.request.clear_response_capture();
}
}
}
impl RequestObservation {
pub(super) fn emit_canonical_response_delivery(
&self,
attempt: &AttemptObservation,
frame_id: &str,
delivery: CanonicalFrameDelivery,
) {
if let Some((store, ticket)) = self.inner.agent_turn_output.get() {
store.accept_output(ticket, &delivery.events);
}
if self.captures_content() {
for event in delivery.events {
let Some(kind) = canonical_response_content_kind(&event.event) else {
continue;
};
let Ok(bytes) = serialize_observed_response_event(&event) else {
self.terminal_content(
"response_delivered",
"abort",
Some("canonical_response_serialization_failed"),
);
return;
};
let part_ordinal = {
let mut state = self.lock_state();
let ordinal = state.response_part_ordinal;
state.response_part_ordinal = ordinal.saturating_add(1);
ordinal
};
let whole_digest =
self.content_blob_digest(CANONICAL_RESPONSE_EVENT_MEDIA_TYPE, &bytes);
let message_instance_id =
self.message_instance_id("response_delivered", 0, "assistant");
let content_ref = ContentRefV1 {
content_id: self.content_id(&message_instance_id, part_ordinal, &whole_digest),
digest: whole_digest.clone(),
byte_count: bytes.len() as u64,
media_type: CANONICAL_RESPONSE_EVENT_MEDIA_TYPE.into(),
};
emit_content_ref(self, "response_delivered", content_ref.clone());
for (index, chunk) in bytes.chunks(CONTENT_CHUNK_BYTES).enumerate() {
self.append_content_chunk(ContentChunk {
direction: "response_delivered",
attempt: Some(attempt),
message_ordinal: 0,
part_ordinal,
role: "assistant",
kind,
media_type: CANONICAL_RESPONSE_EVENT_MEDIA_TYPE,
bytes: chunk,
public_ref: Some(content_ref.clone()),
whole_digest: &whole_digest,
starts_content: index == 0,
transport_frame_id: Some(frame_id),
downstream_delivery: Some("full_frame_transport_accepted"),
});
}
}
}
if let Some(usage) = delivery.usage {
self.usage_model(&usage);
}
}
}
/// Projects execution-internal Tool identity onto the product-visible
/// canonical event. Native IDs and physical ownership remain available to the
/// runtime ledger but never cross the conversation-content port.
fn serialize_observed_response_event(
event: &ModelStreamEventV1,
) -> Result<Vec<u8>, serde_json::Error> {
let mut projected = serde_json::to_value(event)?;
if matches!(
event.event,
ModelEvent::ToolCallStarted { .. } | ModelEvent::WebSearch { .. }
) && let Some(fields) = projected
.get_mut("event")
.and_then(serde_json::Value::as_object_mut)
{
fields.remove("native_id");
fields.remove("owner");
fields.remove("item_id");
}
serde_json::to_vec(&projected)
}
fn canonical_response_content_kind(event: &ModelEvent) -> Option<&'static str> {
match event {
ModelEvent::TextAnnotation { .. } => Some("text_annotation"),
ModelEvent::TextFinished { .. } => Some("text_finished"),
ModelEvent::ReasoningFinished { .. } => Some("reasoning_finished"),
ModelEvent::RefusalFinished { .. } => Some("refusal_finished"),
ModelEvent::WebSearch { .. } => Some("web_search"),
ModelEvent::ContentBlockStarted { .. } => Some("content_block_started"),
ModelEvent::TextDelta { .. } => Some("text_delta"),
ModelEvent::ReasoningDelta { .. } => Some("reasoning_delta"),
ModelEvent::RefusalDelta { .. } => Some("refusal_delta"),
ModelEvent::ToolCallStarted { .. } => Some("tool_call_started"),
ModelEvent::ToolArgumentsDelta { .. } => Some("tool_arguments_delta"),
ModelEvent::ToolCallFinished { .. } => Some("tool_call_finished"),
ModelEvent::ResponseStarted { .. }
| ModelEvent::UsageUpdated { .. }
| ModelEvent::FinishReason { .. }
| ModelEvent::ResponseCompleted { .. }
| ModelEvent::ResponseFailed { .. }
| ModelEvent::ProviderState { .. }
| ModelEvent::ResponsesMetadata { .. } => None,
}
}
impl Drop for AcceptedResponseCapture {
fn drop(&mut self) {
if self.ended {
return;
}
if let Some(capture) = self.request.response_capture() {
capture.cancel_unless_terminal("downstream_transport_incomplete");
}
if let Some(attempt) = &self.attempt {
self.request
.begin_content("response_delivered", Some(attempt));
}
self.request.terminal_content(
"response_delivered",
"abort",
Some("downstream_transport_incomplete"),
);
}
}
struct ContentChunk<'a> {
direction: &'static str,
attempt: Option<&'a AttemptObservation>,
message_ordinal: u32,
part_ordinal: u32,
role: &'a str,
kind: &'a str,
media_type: &'a str,
bytes: &'a [u8],
public_ref: Option<ContentRefV1>,
whole_digest: &'a str,
starts_content: bool,
transport_frame_id: Option<&'a str>,
downstream_delivery: Option<&'a str>,
}
struct ContentEvent {
direction: &'static str,
phase: &'static str,
attempt_id: Option<String>,
fork_id: String,
parent_transcript_root: Option<String>,
result_transcript_root: Option<String>,
message_instance_id: Option<String>,
message_role: Option<String>,
content_kind: Option<String>,
content_id: Option<String>,
content_blob_digest: Option<String>,
message_ordinal: Option<u32>,
part_ordinal: Option<u32>,
chunk_ordinal: Option<u32>,
transport_frame_id: Option<String>,
canonical_media_type: Option<String>,
canonical_bytes_base64: Option<String>,
content_ref: Option<ContentRefV1>,
downstream_delivery: Option<String>,
abort_reason: Option<String>,
completeness_delta: Option<String>,
}
fn update_transcript(transcript: &mut WorkspaceHmac, message_instance_id: &str, content_id: &str) {
transcript.update(message_instance_id.as_bytes());
transcript.update(content_id.as_bytes());
}
fn capture_abort_reason(reason: &str) -> CaptureAbortReason {
match reason {
"canonical_content_unavailable" => CaptureAbortReason::CanonicalContentUnavailable,
"canonical_capture_not_bound" => CaptureAbortReason::CanonicalCaptureNotBound,
"canonical_capture_state_unavailable" => {
CaptureAbortReason::CanonicalCaptureStateUnavailable
}
"canonical_response_serialization_failed" => {
CaptureAbortReason::ResponseSerializationFailed
}
"canonical_response_wire_mismatch"
| "canonical_response_wire_truncated"
| "canonical_response_unexpected_wire_bytes"
| "canonical_response_decode_failed" => CaptureAbortReason::IntegrityFailure,
_ => CaptureAbortReason::Other,
}
}
fn base64_encode(bytes: &[u8]) -> String {
const ALPHABET: &[u8; 64] = b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
let mut output = String::with_capacity(bytes.len().div_ceil(3) * 4);
for chunk in bytes.chunks(3) {
let first = chunk[0];
let second = chunk.get(1).copied().unwrap_or(0);
let third = chunk.get(2).copied().unwrap_or(0);
output.push(ALPHABET[(first >> 2) as usize] as char);
output.push(ALPHABET[(((first & 0x03) << 4) | (second >> 4)) as usize] as char);
output.push(if chunk.len() > 1 {
ALPHABET[(((second & 0x0f) << 2) | (third >> 6)) as usize] as char
} else {
'='
});
output.push(if chunk.len() > 2 {
ALPHABET[(third & 0x3f) as usize] as char
} else {
'='
});
}
output
}
#[cfg(test)]
mod tests {
use serde_json::json;
use crate::server::core_runtime::model_ir::OpaqueProviderState;
use super::*;
#[test]
fn provider_hidden_state_is_not_conversation_response_content() {
let event = ModelEvent::ProviderState {
state: Box::new(OpaqueProviderState {
block_index: Some(0),
kind: "encrypted_content".into(),
value: json!("provider-hidden-secret"),
}),
};
assert_eq!(canonical_response_content_kind(&event), None);
}
}