Skip to content

Make Codex and Claude Code subscriptions go further: a local model router that gives routine agent work to economical models and keeps your strongest model for the hard parts.

18.6 KB
Raw Blame
//! Stateless tool IDs and request-local delivery bookkeeping.
use super::ProtocolAdapterError;
use crate::server::core_runtime::model_ir::{ExactProviderPathV1, ModelIrError};
use crate::server::request_plan::IngressProtocol;
use hiroute_gateway_core::runtime::body::{MemoryRole, Reservation, StreamBudget};
use sha2::{Digest, Sha256};
use std::future::Future;
use std::sync::atomic::{AtomicUsize, Ordering};
use std::sync::{Arc, Mutex};
use std::time::Instant;

/// CPA-style deterministic adaptation, only across protocols. A digest avoids
/// the collisions introduced by replacing arbitrary characters with '_'.
pub(crate) fn project_tool_id(
    id: &str,
    from: IngressProtocol,
    to: IngressProtocol,
) -> Result<String, ModelIrError> {
    if id.trim().is_empty() {
        return Err(ModelIrError::InvalidField("tool id"));
    }
    if from == to {
        return Ok(id.to_owned());
    }
    let invalid = to == IngressProtocol::Messages
        && !id
            .bytes()
            .all(|b| b.is_ascii_alphanumeric() || b == b'_' || b == b'-');
    if !invalid {
        return Ok(id.to_owned());
    }
    let prefix: String = id
        .chars()
        .take(31)
        .map(|c| {
            if c.is_ascii_alphanumeric() || c == '_' || c == '-' {
                c
            } else {
                '_'
            }
        })
        .collect();
    let digest = Sha256::digest(id.as_bytes());
    let suffix: String = digest[..16].iter().map(|b| format!("{b:02x}")).collect();
    Ok(format!("{prefix}_{suffix}"))
}

#[derive(Clone, Debug)]
pub(crate) struct ToolIdProjection {
    downstream: IngressProtocol,
}
impl ToolIdProjection {
    pub(crate) fn new(downstream: IngressProtocol) -> Self {
        Self { downstream }
    }
    pub(crate) fn project(
        &self,
        native_id: &str,
        owner: &ExactProviderPathV1,
    ) -> Result<String, ProtocolAdapterError> {
        Ok(project_tool_id(
            native_id,
            owner.upstream_protocol,
            self.downstream,
        )?)
    }
}

// Counting writer uses the same escaping rules as the actual serializer.
struct JsonLength(usize);

impl std::io::Write for JsonLength {
    fn write(&mut self, bytes: &[u8]) -> std::io::Result<usize> {
        self.0 = self
            .0
            .checked_add(bytes.len())
            .ok_or_else(|| std::io::Error::other("serialized tool ID length overflow"))?;
        Ok(bytes.len())
    }

    fn flush(&mut self) -> std::io::Result<()> {
        Ok(())
    }
}

pub(crate) fn json_string_length(value: &str) -> Result<usize, ModelIrError> {
    let mut length = JsonLength(0);
    serde_json::to_writer(&mut length, value).map_err(|_| ModelIrError::BufferLimit(usize::MAX))?;
    Ok(length.0)
}

struct PendingTool {
    id: String,
    pattern: Vec<u8>,
    charge: Arc<Reservation>,
}

#[derive(Clone)]
pub(crate) struct ActiveResponseDelivery {
    projection: ToolIdProjection,
    budget: StreamBudget,
    pending_tools: Arc<Mutex<Vec<PendingTool>>>,
    accepted_count: Arc<AtomicUsize>,
}
impl ActiveResponseDelivery {
    pub(crate) fn new(downstream: IngressProtocol, budget: StreamBudget) -> Self {
        Self {
            projection: ToolIdProjection::new(downstream),
            budget,
            pending_tools: Default::default(),
            accepted_count: Default::default(),
        }
    }
    pub(crate) fn scanner(&self) -> AcceptedResponseDeliveryScanner {
        AcceptedResponseDeliveryScanner {
            active: self.clone(),
            carry: Vec::new(),
            carry_charge: None,
        }
    }
    pub(crate) fn accepted_count(&self) -> usize {
        self.accepted_count.load(Ordering::Acquire)
    }
    pub(crate) fn tool_id_projection(&self) -> ToolIdProjection {
        self.projection.clone()
    }
}
tokio::task_local! { static ACTIVE_RESPONSE_DELIVERY: ActiveResponseDelivery; }
pub(crate) async fn with_active_response_delivery<F: Future>(
    active: ActiveResponseDelivery,
    future: F,
) -> F::Output {
    ACTIVE_RESPONSE_DELIVERY.scope(active, future).await
}
pub(super) fn project_delivered_tool_id(
    native_id: &str,
    owner: &ExactProviderPathV1,
) -> Result<String, ProtocolAdapterError> {
    let active = ACTIVE_RESPONSE_DELIVERY.try_with(Clone::clone).ok();
    let Some(active) = active else {
        return Ok(native_id.to_owned());
    };
    let id = active.projection.project(native_id, owner)?;
    let mut pending = active
        .pending_tools
        .lock()
        .unwrap_or_else(|e| e.into_inner());
    if !pending.iter().any(|entry| entry.id == id) {
        let prefix: &[u8] = match active.projection.downstream {
            IngressProtocol::Responses => b"\"call_id\":",
            _ => b"\"id\":",
        };
        let length = json_string_length(&id)?
            .checked_add(prefix.len())
            .ok_or(ModelIrError::BufferLimit(usize::MAX))?;
        let bytes = length
            .checked_mul(8)
            .and_then(|n| n.checked_add(std::mem::size_of::<PendingTool>()))
            .ok_or(ModelIrError::BufferLimit(usize::MAX))?;
        let charge = active
            .budget
            .reserve(MemoryRole::SemanticState, bytes)
            .map_err(|_| ModelIrError::BufferLimit(bytes))?;
        let mut pattern = Vec::with_capacity(length);
        pattern.extend_from_slice(prefix);
        serde_json::to_writer(&mut pattern, &id)
            .map_err(|e| ProtocolAdapterError::Serialization(e.to_string()))?;
        pending.push(PendingTool {
            id: id.clone(),
            pattern,
            charge: Arc::new(charge),
        });
    }
    Ok(id)
}
/// Delivery facts only; no later request queries these patterns.
pub(crate) struct AcceptedResponseDeliveryScanner {
    active: ActiveResponseDelivery,
    carry: Vec<u8>,
    carry_charge: Option<Arc<Reservation>>,
}
impl AcceptedResponseDeliveryScanner {
    pub(crate) fn accept_bytes(&mut self, bytes: &[u8], _now: Instant) {
        let mut pending = self
            .active
            .pending_tools
            .lock()
            .unwrap_or_else(|e| e.into_inner());
        let keep = pending
            .iter()
            .map(|p| p.pattern.len().saturating_sub(1))
            .max()
            .unwrap_or(0);
        let next_charge = pending
            .iter()
            .max_by_key(|p| p.pattern.len())
            .map(|p| Arc::clone(&p.charge));
        let mut boundary = self.carry.clone();
        boundary.extend_from_slice(&bytes[..bytes.len().min(keep)]);
        pending.retain(|entry| {
            let pattern = &entry.pattern;
            let accepted = bytes.windows(pattern.len()).any(|w| w == pattern)
                || boundary.windows(pattern.len()).any(|w| w == pattern);
            if accepted {
                self.active.accepted_count.fetch_add(1, Ordering::AcqRel);
            }
            !accepted
        });
        if bytes.len() >= keep {
            self.carry = bytes[bytes.len() - keep..].to_vec();
        } else {
            self.carry.extend_from_slice(bytes);
            let drop = self.carry.len().saturating_sub(keep);
            self.carry.drain(..drop);
        }
        self.carry_charge = next_charge;
    }
}
#[cfg(test)]
mod tests {
    use super::*;
    use crate::server::core_runtime::adapters::{
        ClientResponseRenderer, NativeResponseDecoder, RenderedClientResponse,
        decode_ingress_request, project_candidate_request,
    };
    use crate::server::core_runtime::profiles::{
        CandidateProtocolProfile, ClientProtocolProfile, fixed_reasoning,
    };
    use serde_json::json;

    #[tokio::test]
    async fn tool_delivery_uses_budget_without_fixed_pattern_size_or_count_limits() {
        let budget =
            hiroute_gateway_core::runtime::body::BudgetTree::new(8 * 1024 * 1024, 8 * 1024 * 1024)
                .unwrap()
                .stream(8 * 1024 * 1024)
                .unwrap();
        let active = ActiveResponseDelivery::new(IngressProtocol::Responses, budget.clone());
        let owner = CandidateProtocolProfile::exact_portable_path(
            IngressProtocol::Responses,
            IngressProtocol::Responses,
            "native",
            fixed_reasoning("fixed"),
        )
        .exact_provider_path()
        .unwrap();
        let long = "n".repeat(300 * 1024);
        with_active_response_delivery(active.clone(), async {
            assert_eq!(project_delivered_tool_id(&long, &owner).unwrap(), long);
            for index in 0..129 {
                let id = format!("call-{index}");
                assert_eq!(project_delivered_tool_id(&id, &owner).unwrap(), id);
            }
        })
        .await;
        assert_eq!(active.pending_tools.lock().unwrap().len(), 130);
        let mut scanner = active.scanner();
        let wire = serde_json::to_vec(&json!({"call_id":long})).unwrap();
        for chunk in wire.chunks(8191) {
            scanner.accept_bytes(chunk, Instant::now());
        }
        assert_eq!(active.accepted_count(), 1);
        assert!(budget.snapshot().unwrap().live > 0);
        drop(scanner);
        drop(active);
        assert_eq!(budget.snapshot().unwrap().live, 0);
    }

    #[test]
    fn repeated_native_ids_preserve_each_result_without_history_pairing() {
        let document = json!({"model":"alias","input":[
            {"type":"function_call","call_id":"same","name":"lookup","arguments":"{}"},
            {"type":"function_call_output","call_id":"same","output":"first"},
            {"type":"function_call","call_id":"same","name":"lookup","arguments":"{}"},
            {"type":"function_call_output","call_id":"same","output":"second"}
        ]});
        let request = decode_ingress_request(IngressProtocol::Responses, &document).unwrap();
        for target in [IngressProtocol::Responses, IngressProtocol::Messages] {
            let profile = CandidateProtocolProfile::exact_portable_path(
                IngressProtocol::Responses,
                target,
                "candidate",
                fixed_reasoning("fixed"),
            );
            let wire = project_candidate_request(&request, &profile).unwrap().body;
            if target == IngressProtocol::Responses {
                assert_eq!(wire["input"], document["input"]);
            } else {
                assert_eq!(wire["messages"][1]["content"][0]["content"], "first");
                assert_eq!(wire["messages"][3]["content"][0]["content"], "second");
                assert_eq!(wire["messages"][3]["content"][0]["tool_use_id"], "same");
            }
        }
        // The wire item defines its kind; a same-ID call does not override it.
        let result_only = json!({"model":"alias","input":[
            {"type":"function_call","call_id":"same","namespace":"group","name":"lookup","arguments":"{}"},
            {"type":"custom_tool_call_output","call_id":"same","output":"native output"},
            {"type":"function_call_output","call_id":"unknown","output":"standalone"}
        ]});
        let request = decode_ingress_request(IngressProtocol::Responses, &result_only).unwrap();
        let profile = CandidateProtocolProfile::exact_portable_path(
            IngressProtocol::Responses,
            IngressProtocol::Responses,
            "candidate",
            fixed_reasoning("fixed"),
        );
        assert_eq!(
            project_candidate_request(&request, &profile).unwrap().body["input"],
            result_only["input"]
        );
    }

    #[test]
    fn tool_history_pairs_without_owner_and_rejects_projected_collisions() {
        let id = "call.with-punctuation";
        let projected_id =
            project_tool_id(id, IngressProtocol::Responses, IngressProtocol::Messages).unwrap();
        let mut document = json!({"model":"alias","input":[
            {"type":"function_call","call_id":id,"name":"lookup","arguments":"{}"},
            {"type":"function_call_output","call_id":id,"output":"done"}
        ]});
        let request = decode_ingress_request(IngressProtocol::Responses, &document).unwrap();
        for model in ["first-candidate", "different-candidate"] {
            let profile = CandidateProtocolProfile::exact_portable_path(
                IngressProtocol::Responses,
                IngressProtocol::Messages,
                model,
                fixed_reasoning("fixed"),
            );
            let wire = project_candidate_request(&request, &profile).unwrap().body;
            assert_eq!(wire["messages"][0]["content"][0]["id"], projected_id);
            assert_eq!(
                wire["messages"][1]["content"][0]["tool_use_id"],
                projected_id
            );
        }
        document["input"].as_array_mut().unwrap().push(json!({
            "type":"function_call", "call_id":projected_id, "name":"lookup", "arguments":"{}"
        }));
        let request = decode_ingress_request(IngressProtocol::Responses, &document).unwrap();
        let profile = CandidateProtocolProfile::exact_portable_path(
            IngressProtocol::Responses,
            IngressProtocol::Messages,
            "candidate",
            fixed_reasoning("fixed"),
        );
        assert_eq!(
            project_candidate_request(&request, &profile).unwrap_err(),
            ProtocolAdapterError::ModelIr(ModelIrError::ToolContinuationConflict)
        );
    }

    #[test]
    fn response_long_id_is_stateless_and_returned_history_keeps_the_pair() {
        let id = "native".repeat(20);
        let profile = CandidateProtocolProfile::exact_portable_path(
            IngressProtocol::Responses,
            IngressProtocol::ChatCompletions,
            "candidate",
            fixed_reasoning("fixed"),
        );
        let body = json!({"id":"response","model":"candidate","choices":[{
            "index":0,"message":{"role":"assistant","content":null,"tool_calls":[{
                "id":id,"type":"function","function":{"name":"lookup","arguments":"{}"}
            }]},"finish_reason":"tool_calls"
        }],"usage":{"prompt_tokens":10,"completion_tokens":5,"total_tokens":15}});
        let mut decoder = NativeResponseDecoder::new(&profile, 200, false).unwrap();
        decoder
            .feed(&serde_json::to_vec(&body).unwrap(), true)
            .unwrap();
        let response = decoder.finish().unwrap();
        let wire_id = &response.response.tool_id_map[0].logical_id;
        assert_eq!(wire_id, &"native".repeat(20));
        let client = ClientProtocolProfile::for_candidate(&profile).unwrap();
        let RenderedClientResponse::Json { body, .. } =
            ClientResponseRenderer::render_nonstream_with_profile(
                &client,
                "alias",
                &response.response,
            )
            .unwrap()
        else {
            panic!("expected nonstream response");
        };
        let returned_call = &body["output"][0];
        assert!(returned_call["id"].is_string());
        assert_eq!(returned_call["status"], "completed");
        let request = decode_ingress_request(
            IngressProtocol::Responses,
            &json!({"model":"alias",
                "tools":[{"type":"function","name":"lookup","parameters":{"type":"object"}}],
                "input":[returned_call,
                         {"type":"function_call_output","call_id":wire_id,"output":"done"}]
            }),
        )
        .unwrap();
        let wire = project_candidate_request(&request, &profile).unwrap().body;
        assert_eq!(wire["messages"][0]["tool_calls"][0]["id"], *wire_id);
        assert_eq!(wire["messages"][1]["tool_call_id"], *wire_id);
    }
    #[test]
    fn completed_response_envelopes_are_portable_but_native_metadata_is_not() {
        let completed = json!({"model":"alias",
            "tools":[{"type":"function","name":"lookup","parameters":{"type":"object"}}],
            "input":[
            {"type":"message","role":"assistant","id":"msg_1","status":"completed",
             "content":[{"type":"output_text","text":"using a tool","annotations":[]}]},
            {"type":"function_call","id":"fc_1","status":"completed",
             "call_id":"call_1","name":"lookup","arguments":"{}"},
            {"type":"function_call_output","call_id":"call_1","output":"done"}
        ]});
        for target in [IngressProtocol::ChatCompletions, IngressProtocol::Messages] {
            let profile = CandidateProtocolProfile::exact_portable_path(
                IngressProtocol::Responses,
                target,
                "candidate",
                fixed_reasoning("fixed"),
            );
            let request = decode_ingress_request(IngressProtocol::Responses, &completed).unwrap();
            project_candidate_request(&request, &profile).unwrap();
            for status in ["in_progress", "incomplete", "provider-private"] {
                let mut unfinished = completed.clone();
                unfinished["input"][1]["status"] = json!(status);
                let request =
                    decode_ingress_request(IngressProtocol::Responses, &unfinished).unwrap();
                assert!(
                    project_candidate_request(&request, &profile).is_err(),
                    "{status}"
                );
            }
            let mut phased = completed.clone();
            phased["input"][0]["phase"] = json!("analysis");
            let request = decode_ingress_request(IngressProtocol::Responses, &phased).unwrap();
            assert!(project_candidate_request(&request, &profile).is_err());
            let mut instruction = completed.clone();
            instruction["input"][0] = json!({"type":"message","role":"system","id":"sys_1",
                "status":"completed","content":"native instruction identity"});
            let request = decode_ingress_request(IngressProtocol::Responses, &instruction).unwrap();
            assert!(project_candidate_request(&request, &profile).is_err());
        }
    }

    #[test]
    fn native_id_and_stateless_cross_protocol_adaptation() {
        use IngressProtocol::*;
        for protocol in [Responses, Messages, ChatCompletions] {
            assert_eq!(
                project_tool_id("native-call-1", protocol, protocol).unwrap(),
                "native-call-1"
            );
            assert_eq!(
                project_tool_id("native-call-1", protocol, Responses).unwrap(),
                "native-call-1"
            );
        }
        let long = "x".repeat(100);
        let shortened = project_tool_id(&long, Messages, Responses).unwrap();
        assert_eq!(shortened, long);
        assert_eq!(
            project_tool_id(&shortened, Messages, Responses).unwrap(),
            shortened
        );
        assert_ne!(
            project_tool_id("call.a", Responses, Messages).unwrap(),
            project_tool_id("call a", Responses, Messages).unwrap()
        );
        assert_eq!(
            project_tool_id("call.a", Responses, Responses).unwrap(),
            "call.a"
        );
    }
}