Skip to content

Make Codex and Claude Code subscriptions go further: a local model router that gives routine agent work to economical models and keeps your strongest model for the hard parts.

24.3 KB
Raw Blame
use std::io::Write;

use serde::{Deserialize, Serialize};
use serde_json::{Map, Value};

use crate::replay::{ReplayContentWriter, ReplayError, ReplayStore};
use crate::server::core_runtime::model_ir::{
    ContentPart, ImageSource, ModelRequestIRV1, ToolOutput,
};

pub(crate) const MARKER_PREFIX: &str = "__hiroute_content_ref_v2_";
const MARKER_SUFFIX: &str = "__";
const JSON_MARKER_KEY: &str = "__hiroute_json_content_ref_v2";
const CONTENT_FIELD_METADATA_BYTES: usize = 192;

mod ingress;

#[cfg(test)]
pub(crate) use ingress::compact_ingress_document;
pub(crate) use ingress::{
    compact_ingress_document_with_markers, parse_ingress_document, scan_ingress_document,
};

/// Stable, path-free locator for one range in the request's replay owner. It
/// deliberately carries no content digest, key, file name, or runtime session
/// identifier and is useful only together with that request-local owner.
#[derive(Clone, Debug, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct ContentRef {
    stream_ordinal: u64,
    byte_offset: u64,
    byte_len: u64,
    json_escaped_len: u64,
}

impl ContentRef {
    pub(crate) fn new(
        stream_ordinal: u64,
        byte_offset: u64,
        byte_len: u64,
        json_escaped_len: u64,
    ) -> Self {
        Self {
            stream_ordinal,
            byte_offset,
            byte_len,
            json_escaped_len,
        }
    }

    pub fn stream_ordinal(&self) -> u64 {
        self.stream_ordinal
    }

    pub fn byte_len(&self) -> u64 {
        self.byte_len
    }

    pub fn byte_offset(&self) -> u64 {
        self.byte_offset
    }

    pub fn json_escaped_len(&self) -> u64 {
        self.json_escaped_len
    }

    /// Compact structural token held by the Model IR after the corresponding
    /// large string moves into Replay. The request-local owner validates the
    /// complete reference before opening a reader, so a forged token can only
    /// fail closed.
    pub(crate) fn wire_marker(&self) -> String {
        format!(
            "{MARKER_PREFIX}{}_{}_{}_{}{MARKER_SUFFIX}",
            self.stream_ordinal, self.byte_offset, self.byte_len, self.json_escaped_len
        )
    }

    pub(crate) fn from_wire_marker(value: &str) -> Option<Self> {
        let payload = value
            .strip_prefix(MARKER_PREFIX)?
            .strip_suffix(MARKER_SUFFIX)?;
        let mut fields = payload.split('_');
        let stream_ordinal = fields.next()?.parse().ok()?;
        let byte_offset = fields.next()?.parse().ok()?;
        let byte_len = fields.next()?.parse().ok()?;
        let json_escaped_len = fields.next()?.parse().ok()?;
        if fields.next().is_some() {
            return None;
        }
        Some(Self::new(
            stream_ordinal,
            byte_offset,
            byte_len,
            json_escaped_len,
        ))
    }

    pub(crate) fn json_marker(&self) -> Value {
        let mut marker = Map::new();
        marker.insert(JSON_MARKER_KEY.into(), Value::String(self.wire_marker()));
        Value::Object(marker)
    }
}

/// Preserve the existing public Model IR construction surface (`String`) while
/// representing externalized values as compact, stable ContentRef tokens.
/// This avoids forcing unrelated request producers to know replay internals.
pub type ContentValue = String;

pub(crate) trait ContentValueExt {
    fn content_ref(&self) -> Option<ContentRef>;
    fn wire_value(&self) -> String;
}

pub(crate) trait JsonValueExt {
    fn content_ref(&self) -> Option<ContentRef>;
    fn wire_value(&self) -> Value;
}

impl ContentValueExt for str {
    fn content_ref(&self) -> Option<ContentRef> {
        ContentRef::from_wire_marker(self)
    }

    fn wire_value(&self) -> String {
        self.to_owned()
    }
}

impl JsonValueExt for Value {
    fn content_ref(&self) -> Option<ContentRef> {
        let object = self.as_object()?;
        if object.len() != 1 {
            return None;
        }
        ContentRef::from_wire_marker(object.get(JSON_MARKER_KEY)?.as_str()?)
    }

    fn wire_value(&self) -> Value {
        if let Some(content) = self.content_ref() {
            return Value::String(content.wire_marker());
        }
        match self {
            Value::Array(values) => {
                Value::Array(values.iter().map(JsonValueExt::wire_value).collect())
            }
            Value::Object(values) => Value::Object(
                values
                    .iter()
                    .map(|(key, value)| (key.clone(), value.wire_value()))
                    .collect(),
            ),
            _ => self.clone(),
        }
    }
}

/// Moves large canonical strings into the same request-owned replay backing.
/// The raw ingress stream is still available to the core body lifecycle until
/// EOS; it is released immediately afterwards, leaving only canonical refs for
/// long responses.
pub fn externalize_model_request(
    request: &mut ModelRequestIRV1,
    replay: &ReplayStore,
    inline_limit: usize,
) -> Result<(), ReplayError> {
    let field_count = content_field_count(request)?;
    replay.charge_metadata(
        field_count
            .checked_mul(CONTENT_FIELD_METADATA_BYTES)
            .ok_or(ReplayError::LengthOverflow)?,
    )?;
    // This is one aggregate allowance, not a per-field threshold. Otherwise a
    // request can split a large prompt across many individually-small fields
    // and make every Attempt rebuild a full-size structural wire buffer.
    let mut inline_remaining = inline_limit;
    let mut pool = None;
    for instruction in &mut request.instructions {
        externalize_parts(
            &mut instruction.content,
            replay,
            &mut pool,
            &mut inline_remaining,
        )?;
    }
    for message in &mut request.messages {
        if let Some(name) = &mut message.name {
            externalize_value(name, replay, &mut pool, &mut inline_remaining)?;
        }
        externalize_parts(
            &mut message.content,
            replay,
            &mut pool,
            &mut inline_remaining,
        )?;
    }
    for history in request.responses_reasoning_history.values_mut() {
        for (key, value) in &mut history.native_fields {
            // Cross-protocol projections still need the plaintext leaves. Keep
            // their small structural envelope, not a raw-JSON ref to the array.
            if matches!(key.as_str(), "summary" | "content")
                && let Some(parts) = value.as_array_mut()
            {
                for part in parts {
                    if let Some(fields) = part.as_object_mut() {
                        for value in fields.values_mut() {
                            if !contains_replay_string_ref(value, replay) {
                                externalize_json(
                                    value,
                                    replay,
                                    &mut pool,
                                    &mut inline_remaining,
                                    false,
                                )?;
                            }
                        }
                    }
                }
                continue;
            }
            if !contains_replay_string_ref(value, replay) {
                externalize_json(value, replay, &mut pool, &mut inline_remaining, false)?;
            }
        }
    }
    // Tool kind/name/namespace values are request-scoped identity keys. Keep
    // them directly comparable across declarations, choices, calls, results,
    // continuation bindings and candidate projections; independent replay
    // markers for equal strings are intentionally not identity-equivalent.
    for tool in &mut request.tools {
        if let Some(description) = &mut tool.description {
            externalize_value(description, replay, &mut pool, &mut inline_remaining)?;
        }
        for value in [&mut tool.input_schema, &mut tool.format]
            .into_iter()
            .flatten()
        {
            externalize_json(value, replay, &mut pool, &mut inline_remaining, false)?;
        }
    }
    for namespace in &mut request.tool_namespaces {
        if let Some(description) = &mut namespace.description {
            externalize_value(description, replay, &mut pool, &mut inline_remaining)?;
        }
        for tool in &mut namespace.tools {
            if let Some(description) = &mut tool.description {
                externalize_value(description, replay, &mut pool, &mut inline_remaining)?;
            }
            for value in [&mut tool.input_schema, &mut tool.format]
                .into_iter()
                .flatten()
            {
                externalize_json(value, replay, &mut pool, &mut inline_remaining, false)?;
            }
        }
    }
    if let Some(value) = &mut request.requested_reasoning.native_value {
        externalize_json(value, replay, &mut pool, &mut inline_remaining, false)?;
    }
    for state in &mut request.provider_state {
        if !contains_replay_string_ref(&state.value, replay) {
            externalize_json(
                &mut state.value,
                replay,
                &mut pool,
                &mut inline_remaining,
                false,
            )?;
        }
    }
    if let Some(body) = &mut request.native_body {
        externalize_native_leaves(body, replay, &mut pool, &mut inline_remaining)?;
    }
    if let Some(pool) = pool {
        pool.seal()?;
    }
    Ok(())
}

pub fn model_content_refs(request: &ModelRequestIRV1) -> Vec<ContentRef> {
    let mut refs = Vec::new();
    if let Some(body) = &request.native_body {
        collect_nested_content_refs(body, &mut refs);
    }
    for instruction in &request.instructions {
        collect_part_content_refs(&instruction.content, &mut refs);
    }
    for message in &request.messages {
        if let Some(content) = message
            .name
            .as_deref()
            .and_then(ContentValueExt::content_ref)
        {
            refs.push(content);
        }
        collect_part_content_refs(&message.content, &mut refs);
    }
    for history in request.responses_reasoning_history.values() {
        for value in history.native_fields.values() {
            collect_nested_content_refs(value, &mut refs);
        }
    }
    for tool in &request.tools {
        if let Some(description) = &tool.description {
            collect_string_content_ref(description, &mut refs);
        }
        if let Some(value) = &tool.input_schema {
            collect_json_content_ref(value, &mut refs);
        }
        if let Some(value) = &tool.format {
            collect_json_content_ref(value, &mut refs);
        }
    }
    for namespace in &request.tool_namespaces {
        if let Some(description) = &namespace.description {
            collect_string_content_ref(description, &mut refs);
        }
        for tool in &namespace.tools {
            if let Some(description) = &tool.description {
                collect_string_content_ref(description, &mut refs);
            }
            if let Some(value) = &tool.input_schema {
                collect_json_content_ref(value, &mut refs);
            }
            if let Some(value) = &tool.format {
                collect_json_content_ref(value, &mut refs);
            }
        }
    }
    if let Some(value) = &request.requested_reasoning.native_value {
        collect_json_content_ref(value, &mut refs);
    }
    for state in &request.provider_state {
        collect_nested_content_refs(&state.value, &mut refs);
    }
    refs
}

// Keep native structure editable for bounded prefix cleanup, while sharing all
// large string storage with the existing request-local Replay owner.
fn externalize_native_leaves(
    value: &mut Value,
    replay: &ReplayStore,
    pool: &mut Option<ReplayContentWriter>,
    remaining: &mut usize,
) -> Result<(), ReplayError> {
    // Each Value is charged once. Map keys/node overhead belong to the parent;
    // charging a full map slot for array elements double-counts the Value.
    replay.charge_metadata(std::mem::size_of::<Value>())?;
    if value.content_ref().is_some() {
        return Ok(());
    }
    match value {
        Value::String(s) => externalize_value(s, replay, pool, remaining)?,
        Value::Array(values) => {
            for value in values {
                externalize_native_leaves(value, replay, pool, remaining)?;
            }
        }
        Value::Object(values) => {
            for (key, value) in values {
                replay.charge_metadata(key.capacity() + std::mem::size_of::<String>() + 32)?;
                if !matches!(key.as_str(), "type" | "role") {
                    externalize_native_leaves(value, replay, pool, remaining)?;
                } else {
                    replay.charge_metadata(std::mem::size_of::<Value>())?;
                    if let Some(value) = value.as_str() {
                        replay.charge_metadata(value.len())?;
                    }
                }
            }
        }
        _ => {}
    }
    Ok(())
}
fn content_field_count(request: &ModelRequestIRV1) -> Result<usize, ReplayError> {
    let mut count = 0_usize;
    for instruction in &request.instructions {
        count = count
            .checked_add(part_field_count(&instruction.content)?)
            .ok_or(ReplayError::LengthOverflow)?;
    }
    for message in &request.messages {
        let parts = part_field_count(&message.content)?;
        count = count
            .checked_add(usize::from(message.name.is_some()))
            .and_then(|value| value.checked_add(parts))
            .ok_or(ReplayError::LengthOverflow)?;
    }
    count = request
        .responses_reasoning_history
        .values()
        .try_fold(count, |count, history| {
            count
                .checked_add(history.native_fields.iter().try_fold(
                    0usize,
                    |total, (key, value)| {
                        let fields = if matches!(key.as_str(), "summary" | "content") {
                            value.as_array().map_or(1, |parts| {
                                parts
                                    .iter()
                                    .map(|part| part.as_object().map_or(1, |fields| fields.len()))
                                    .sum()
                            })
                        } else {
                            1
                        };
                        total.checked_add(fields).ok_or(ReplayError::LengthOverflow)
                    },
                )?)
                .ok_or(ReplayError::LengthOverflow)
        })?;
    for tool in &request.tools {
        count = count
            .checked_add(
                1 + usize::from(tool.description.is_some())
                    + usize::from(tool.input_schema.is_some())
                    + usize::from(tool.format.is_some()),
            )
            .ok_or(ReplayError::LengthOverflow)?;
    }
    for namespace in &request.tool_namespaces {
        count = count
            .checked_add(1 + usize::from(namespace.description.is_some()))
            .ok_or(ReplayError::LengthOverflow)?;
        for tool in &namespace.tools {
            count = count
                .checked_add(
                    1 + usize::from(tool.description.is_some())
                        + usize::from(tool.input_schema.is_some())
                        + usize::from(tool.format.is_some()),
                )
                .ok_or(ReplayError::LengthOverflow)?;
        }
    }
    count = count
        .checked_add(usize::from(matches!(
            request.tool_choice,
            crate::server::core_runtime::model_ir::ToolChoice::RequiredNamed { .. }
        )))
        .and_then(|value| {
            value.checked_add(usize::from(
                request.requested_reasoning.native_value.is_some(),
            ))
        })
        .and_then(|value| value.checked_add(request.provider_state.len()))
        .ok_or(ReplayError::LengthOverflow)?;
    Ok(count)
}

fn part_field_count(parts: &[ContentPart]) -> Result<usize, ReplayError> {
    parts.iter().try_fold(0_usize, |count, part| {
        count
            .checked_add(match part {
                ContentPart::ToolCall { namespace, .. } => 2 + usize::from(namespace.is_some()),
                ContentPart::ToolResult { .. } => 1,
                ContentPart::ProviderState { .. } => 1,
                ContentPart::Text { .. } | ContentPart::Image { .. } => 1,
            })
            .ok_or(ReplayError::LengthOverflow)
    })
}

fn collect_part_content_refs(parts: &[ContentPart], refs: &mut Vec<ContentRef>) {
    for part in parts {
        match part {
            ContentPart::Text { text } => collect_string_content_ref(text, refs),
            ContentPart::Image {
                source: ImageSource::Url { url },
            } => collect_string_content_ref(url, refs),
            ContentPart::Image {
                source: ImageSource::Base64 { data, .. },
            } => collect_string_content_ref(data, refs),
            ContentPart::ToolCall {
                arguments,
                raw_arguments,
                ..
            } => {
                if let Some(raw) = raw_arguments {
                    collect_string_content_ref(raw, refs);
                } else {
                    collect_json_content_ref(arguments, refs);
                }
            }
            ContentPart::ToolResult {
                output: ToolOutput::Text(value),
                ..
            } => collect_string_content_ref(value, refs),
            ContentPart::ToolResult {
                output: ToolOutput::Json(value),
                ..
            } => collect_json_content_ref(value, refs),
            ContentPart::ProviderState { state } => {
                collect_nested_content_refs(&state.value, refs);
            }
        }
    }
}

fn collect_string_content_ref(value: &str, refs: &mut Vec<ContentRef>) {
    if let Some(content) = value.content_ref() {
        refs.push(content);
    }
}

fn collect_json_content_ref(value: &Value, refs: &mut Vec<ContentRef>) {
    if let Some(content) = value.content_ref() {
        refs.push(content);
    }
}

fn collect_nested_content_refs(value: &Value, refs: &mut Vec<ContentRef>) {
    if let Some(content) = value.content_ref() {
        refs.push(content);
        return;
    }
    match value {
        Value::String(value) => collect_string_content_ref(value, refs),
        Value::Array(values) => {
            for value in values {
                collect_nested_content_refs(value, refs);
            }
        }
        Value::Object(values) => {
            for value in values.values() {
                collect_nested_content_refs(value, refs);
            }
        }
        _ => {}
    }
}

fn contains_replay_string_ref(value: &Value, replay: &ReplayStore) -> bool {
    match value {
        Value::String(value) => ContentRef::from_wire_marker(value)
            .as_ref()
            .is_some_and(|reference| replay.has_reference(reference)),
        Value::Array(values) => values
            .iter()
            .any(|value| contains_replay_string_ref(value, replay)),
        Value::Object(values) => values
            .values()
            .any(|value| contains_replay_string_ref(value, replay)),
        _ => false,
    }
}

fn externalize_parts(
    parts: &mut [ContentPart],
    replay: &ReplayStore,
    pool: &mut Option<ReplayContentWriter>,
    inline_remaining: &mut usize,
) -> Result<(), ReplayError> {
    for part in parts {
        match part {
            ContentPart::Text { text } => externalize_value(text, replay, pool, inline_remaining)?,
            ContentPart::Image {
                source: ImageSource::Url { url },
            } => externalize_value(url, replay, pool, inline_remaining)?,
            ContentPart::Image {
                source: ImageSource::Base64 { data, .. },
            } => externalize_value(data, replay, pool, inline_remaining)?,
            ContentPart::ToolCall {
                arguments,
                raw_arguments,
                ..
            } => match raw_arguments {
                Some(raw) => externalize_value(raw, replay, pool, inline_remaining)?,
                None => externalize_json(arguments, replay, pool, inline_remaining, false)?,
            },
            ContentPart::ToolResult {
                output: ToolOutput::Text(value),
                ..
            } => externalize_value(value, replay, pool, inline_remaining)?,
            ContentPart::ToolResult {
                output: ToolOutput::Json(value),
                ..
            } => externalize_json(value, replay, pool, inline_remaining, true)?,
            ContentPart::ProviderState { state } => {
                // A native opaque block can already contain Replay-backed
                // leaves. The sequential template expands them directly.
                if matches!(state.kind.as_str(), "thinking" | "redacted_thinking") {
                    if let Some(object) = state.value.as_object_mut() {
                        for (key, value) in object {
                            if key != "type" && !contains_replay_string_ref(value, replay) {
                                externalize_json(value, replay, pool, inline_remaining, false)?;
                            }
                        }
                    }
                } else if !contains_replay_string_ref(&state.value, replay) {
                    externalize_json(&mut state.value, replay, pool, inline_remaining, false)?;
                }
            }
        }
    }
    Ok(())
}

fn externalize_value(
    value: &mut ContentValue,
    replay: &ReplayStore,
    pool: &mut Option<ReplayContentWriter>,
    inline_remaining: &mut usize,
) -> Result<(), ReplayError> {
    if ContentRef::from_wire_marker(value)
        .as_ref()
        .is_some_and(|reference| replay.has_reference(reference))
    {
        return Ok(());
    }
    // The marker namespace is reserved. Externalize marker-containing user
    // content even when small so it cannot alias a ref created for an earlier
    // field in the same request.
    if value.len() <= *inline_remaining && !value.contains(MARKER_PREFIX) {
        *inline_remaining -= value.len();
        return Ok(());
    }
    let content_ref = append_to_pool(replay, pool, value.as_bytes())?;
    *value = content_ref.wire_marker();
    Ok(())
}

fn externalize_json(
    value: &mut Value,
    replay: &ReplayStore,
    pool: &mut Option<ReplayContentWriter>,
    inline_remaining: &mut usize,
    canonical: bool,
) -> Result<(), ReplayError> {
    if value
        .content_ref()
        .as_ref()
        .is_some_and(|reference| replay.has_reference(reference))
    {
        return Ok(());
    }
    // JSON-bearing fields can be arbitrarily large. Serialize them directly
    // into the aggregate replay stream instead of first building an ordered
    // clone and a second full Vec. Scalar JSON stays inline while the shared
    // allowance remains; compound JSON is always externalized so Planner and
    // every candidate hold only a compact marker. The marker namespace is
    // reserved for scalar strings here too, for the same reason as ordinary
    // ContentValue fields above.
    if !value.is_array() && !value.is_object() {
        let encoded_len = serialized_json_len(value)?;
        let marker_free = value
            .as_str()
            .is_none_or(|string| !string.contains(MARKER_PREFIX));
        if encoded_len <= *inline_remaining && marker_free {
            *inline_remaining -= encoded_len;
            return Ok(());
        }
    }
    if pool.is_none() {
        *pool = Some(replay.begin_content_pool()?);
    }
    let reference = pool
        .as_mut()
        .ok_or(ReplayError::Integrity)?
        .append_json(value, canonical)?;
    *value = reference.json_marker();
    Ok(())
}

fn serialized_json_len(value: &Value) -> Result<usize, ReplayError> {
    let mut counter = CountingWriter(0);
    serde_json::to_writer(&mut counter, value).map_err(|_| ReplayError::InvalidJson)?;
    Ok(counter.0)
}

struct CountingWriter(usize);

impl Write for CountingWriter {
    fn write(&mut self, bytes: &[u8]) -> std::io::Result<usize> {
        self.0 = self
            .0
            .checked_add(bytes.len())
            .ok_or_else(|| std::io::Error::other("JSON length overflow"))?;
        Ok(bytes.len())
    }

    fn flush(&mut self) -> std::io::Result<()> {
        Ok(())
    }
}

fn append_to_pool(
    replay: &ReplayStore,
    pool: &mut Option<ReplayContentWriter>,
    bytes: &[u8],
) -> Result<ContentRef, ReplayError> {
    if pool.is_none() {
        *pool = Some(replay.begin_content_pool()?);
    }
    pool.as_mut().ok_or(ReplayError::Integrity)?.append(bytes)
}