Skip to content

Make Codex and Claude Code subscriptions go further: a local model router that gives routine agent work to economical models and keeps your strongest model for the hard parts.

21.8 KB
Raw Blame
use std::collections::{BTreeMap, BTreeSet};
use std::mem::size_of;
use std::sync::Arc;
use std::time::{Duration, Instant};

use hiroute_gateway_core::core::execution_plan::ResolvedTargetBindingId;
use hmac::{Hmac, Mac};
use parking_lot::Mutex;
use sha2::Sha256;

use crate::server::core_runtime::profiles::HoldPreferenceV1;
use crate::server::request_plan::{AuthorizedRequestPlan, IngressProtocol};

use super::{ContextIdentityFacts, HistoryEvidence};

type HmacSha256 = Hmac<Sha256>;

pub(crate) const DEFAULT_MAX_MEMORY_BYTES: usize = 50_000_000;
pub(crate) const DEFAULT_IDLE_TTL: Duration = Duration::from_secs(60 * 60);

#[derive(Clone, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)]
pub(crate) struct ContextHoldKey([u8; 32]);

impl ContextHoldKey {
    pub(crate) fn from_request(
        authorized: &AuthorizedRequestPlan,
        ingress_protocol: IngressProtocol,
        identity: &ContextIdentityFacts,
        digest_key: &[u8; 32],
    ) -> Option<Self> {
        let (identity_kind, identity_parts) = identity.hold_identity()?;
        let receipt = authorized.receipt();
        let mut mac = HmacSha256::new_from_slice(digest_key).ok()?;
        frame(&mut mac, b"domain", b"context-hold/v1");
        frame(&mut mac, b"workspace_id", receipt.workspace_id.as_bytes());
        frame(&mut mac, b"authority_id", receipt.authority_id.as_bytes());
        frame_u64(&mut mac, b"authority_epoch", receipt.authority_epoch);
        frame(&mut mac, b"grant_id", receipt.grant_id.as_bytes());
        frame_u64(&mut mac, b"grant_generation", receipt.grant_generation);
        frame(
            &mut mac,
            b"route_identity",
            &serde_json::to_vec(&authorized.planner_policy().identity).ok()?,
        );
        frame(
            &mut mac,
            b"served_model_id",
            receipt.served_model_id.as_bytes(),
        );
        frame(
            &mut mac,
            b"route_provenance",
            &serde_json::to_vec(&receipt.route).ok()?,
        );
        // The aggregate revision also changes for unrelated plans. Continuity
        // belongs to this authorized route/version; current authorization and
        // the exact held candidate/profile are still checked on every request.
        frame(
            &mut mac,
            b"ingress_protocol",
            ingress_label(ingress_protocol),
        );
        frame(&mut mac, b"identity_kind", identity_kind.as_bytes());
        frame_u64(
            &mut mac,
            b"identity_parts_len",
            u64::try_from(identity_parts.len()).ok()?,
        );
        for part in identity_parts {
            frame(&mut mac, b"identity-part", part.as_bytes());
        }
        Some(Self(mac.finalize().into_bytes().into()))
    }
}

fn ingress_label(ingress: IngressProtocol) -> &'static [u8] {
    match ingress {
        IngressProtocol::Responses => b"responses",
        IngressProtocol::ChatCompletions => b"chat_completions",
        IngressProtocol::Messages => b"messages",
    }
}

fn frame(mac: &mut HmacSha256, kind: &[u8], value: &[u8]) {
    mac.update(&(kind.len() as u64).to_be_bytes());
    mac.update(kind);
    mac.update(&(value.len() as u64).to_be_bytes());
    mac.update(value);
}

fn frame_u64(mac: &mut HmacSha256, kind: &[u8], value: u64) {
    frame(mac, kind, &value.to_be_bytes());
}

/// Continuity of visible messages, independent of system-instruction changes.
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub(crate) enum HistoryContinuity {
    Continuing,
    Shortened,
    Rewritten,
    NoCheckpoint,
}

#[derive(Clone, Debug)]
pub(crate) struct HoldTicket {
    key: ContextHoldKey,
    entry_token: u64,
    cycle: u64,
    value_version: u64,
    checkpoint_revision: u64,
    pub(crate) history_continuity: HistoryContinuity,
    pub(crate) hint: Option<HoldPreferenceV1>,
    /// Last fully delivered candidate in this exact route scope, even when
    /// rebuilt history invalidates the stronger continuity hint.
    pub(crate) previous_success: Option<HoldPreferenceV1>,
    pub(crate) cleaned_prefix_len: Option<usize>,
}

#[derive(Clone, Debug)]
pub(crate) struct HoldCompletion {
    pub(crate) ticket: HoldTicket,
    pub(crate) candidates: Arc<[(ResolvedTargetBindingId, HoldPreferenceV1)]>,
}

impl HoldCompletion {
    pub(crate) fn preference_for(
        &self,
        binding: ResolvedTargetBindingId,
    ) -> Option<&HoldPreferenceV1> {
        self.candidates
            .iter()
            .find_map(|(candidate_binding, preference)| {
                (*candidate_binding == binding).then_some(preference)
            })
    }
}

#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub(crate) enum HoldCompleteOutcome {
    Applied,
    Stale,
    CapacityMiss,
}

pub(crate) struct ContextHoldStore {
    key: Option<[u8; 32]>,
    max_memory_bytes: usize,
    idle_ttl: Duration,
    inner: Mutex<StoreInner>,
}

struct StoreInner {
    entries: BTreeMap<ContextHoldKey, HoldEntry>,
    lru: BTreeSet<LruRecord>,
    next_token: u64,
    access_clock: u64,
    dynamic_bytes: usize,
    peak_accounted_bytes: usize,
    #[cfg(test)]
    maintenance_visits: usize,
}

struct HoldEntry {
    entry_token: u64,
    cycle: u64,
    checkpoint_revision: u64,
    value_version: u64,
    instruction_digest: [u8; 32],
    message_count: usize,
    history_digest: [u8; 32],
    preference: Option<HoldPreferenceV1>,
    hint_valid: bool,
    cleaned_prefix_len: Option<usize>,
    last_access: Instant,
    access_order: u64,
}

#[derive(Clone, Debug, Eq, Ord, PartialEq, PartialOrd)]
struct LruRecord {
    last_access: Instant,
    access_order: u64,
    key: ContextHoldKey,
}

#[derive(Clone, Copy)]
struct BeginSnapshot {
    entry_token: u64,
    cycle: u64,
    checkpoint_revision: u64,
    instruction_digest: [u8; 32],
    message_count: usize,
    history_digest: [u8; 32],
}

const MAINTENANCE_BATCH: usize = 16;
const EVICTION_BATCH: usize = 64;
const TREE_INDEX_BASE_BYTES: usize = 2_048;
const TREE_INDEX_ENTRY_BYTES: usize = 1_024;

impl Default for ContextHoldStore {
    fn default() -> Self {
        Self::new(DEFAULT_MAX_MEMORY_BYTES, DEFAULT_IDLE_TTL)
    }
}

impl ContextHoldStore {
    pub(crate) fn new(max_memory_bytes: usize, idle_ttl: Duration) -> Self {
        let mut key = [0_u8; 32];
        let key = getrandom::fill(&mut key).ok().map(|()| key);
        Self {
            key,
            max_memory_bytes,
            idle_ttl,
            inner: Mutex::new(StoreInner {
                entries: BTreeMap::new(),
                lru: BTreeSet::new(),
                next_token: 1,
                access_clock: 0,
                dynamic_bytes: 0,
                peak_accounted_bytes: store_fixed_bytes(),
                #[cfg(test)]
                maintenance_visits: 0,
            }),
        }
    }

    pub(crate) fn digest_key(&self) -> Option<[u8; 32]> {
        self.key
    }

    pub(crate) fn begin(
        &self,
        key: ContextHoldKey,
        history: &HistoryEvidence<'_>,
        now: Instant,
    ) -> Option<HoldTicket> {
        self.begin_with_measure(key, history, now, |prefix| history.measure(prefix))
    }

    fn begin_with_measure(
        &self,
        key: ContextHoldKey,
        history: &HistoryEvidence<'_>,
        now: Instant,
        measure: impl FnOnce(Option<usize>) -> Option<super::history::HistoryDigests>,
    ) -> Option<HoldTicket> {
        self.key?;
        let snapshot = {
            let mut inner = self.inner.lock();
            inner.purge_expired_bounded(now, self.idle_ttl);
            inner.remove_if_expired(&key, now, self.idle_ttl);
            inner.entries.get(&key).map(BeginSnapshot::from)
        };

        // Scan visible history outside the mutex, retaining only the stored
        // prefix checkpoint and the final digest.
        let measured = measure(snapshot.map(|entry| entry.message_count))?;
        let mut inner = self.inner.lock();
        inner.remove_if_expired(&key, now, self.idle_ttl);
        if let Some(snapshot) = snapshot {
            let (current_cycle, current_value_version) = {
                let entry = inner.entries.get(&key)?;
                if !snapshot.matches_checkpoint(entry) {
                    // A newer entrance won the checkpoint CAS. This request
                    // uses the ordinary route and never overwrites history.
                    return None;
                }
                (entry.cycle, entry.value_version)
            };
            let message_history_continues = measured.message_count >= snapshot.message_count
                && measured.prefix_digest == Some(snapshot.history_digest);
            let continues = snapshot.instruction_digest == history.instruction_digest
                && message_history_continues;
            let checkpoint_revision = snapshot.checkpoint_revision.checked_add(1)?;
            let cycle = if continues {
                current_cycle
            } else {
                current_cycle.checked_add(1)?
            };
            let value_version = if continues {
                current_value_version
            } else {
                current_value_version.checked_add(1)?
            };
            let access_order = inner.allocate_access_order()?;
            let (hint, previous_success, cleaned_prefix_len, removed_dynamic) = {
                let entry = inner.entries.get_mut(&key)?;
                let previous_success = entry.preference.clone();
                let hint = (continues && entry.hint_valid)
                    .then(|| entry.preference.clone())
                    .flatten();
                let cleaned_prefix_len = message_history_continues
                    .then_some(entry.cleaned_prefix_len)
                    .flatten();
                let removed_dynamic = if message_history_continues {
                    0
                } else {
                    entry.preference.take().as_ref().map_or(0, preference_bytes)
                };
                entry.cycle = cycle;
                entry.checkpoint_revision = checkpoint_revision;
                entry.value_version = value_version;
                entry.instruction_digest = history.instruction_digest;
                entry.message_count = measured.message_count;
                entry.history_digest = measured.complete_digest;
                entry.hint_valid &= continues;
                entry.cleaned_prefix_len = cleaned_prefix_len;
                (hint, previous_success, cleaned_prefix_len, removed_dynamic)
            };
            inner.dynamic_bytes = inner.dynamic_bytes.saturating_sub(removed_dynamic);
            inner.touch(&key, now, access_order);
            return Some(HoldTicket {
                key,
                entry_token: snapshot.entry_token,
                cycle,
                value_version,
                checkpoint_revision,
                history_continuity: if message_history_continues {
                    HistoryContinuity::Continuing
                } else if measured.message_count < snapshot.message_count {
                    HistoryContinuity::Shortened
                } else {
                    HistoryContinuity::Rewritten
                },
                hint,
                previous_success,
                cleaned_prefix_len,
            });
        }

        if inner.entries.contains_key(&key) {
            // Another request inserted this scope while the history was being
            // hashed. Do not retry the scan under contention.
            return None;
        }
        let entry_token = inner.allocate_entry_token()?;
        let access_order = inner.allocate_access_order()?;
        let entry = HoldEntry {
            entry_token,
            cycle: 1,
            checkpoint_revision: 1,
            value_version: 0,
            instruction_digest: history.instruction_digest,
            message_count: measured.message_count,
            history_digest: measured.complete_digest,
            preference: None,
            hint_valid: false,
            cleaned_prefix_len: None,
            last_access: now,
            access_order,
        };
        if !inner.make_room_for_insert(&entry, self.max_memory_bytes) {
            return None;
        }
        inner.entries.insert(key.clone(), entry);
        inner.lru.insert(LruRecord {
            last_access: now,
            access_order,
            key: key.clone(),
        });
        let accounted = inner.accounted_bytes();
        inner.peak_accounted_bytes = inner.peak_accounted_bytes.max(accounted);
        if accounted > self.max_memory_bytes {
            inner.remove_entry(&key);
            return None;
        }
        Some(HoldTicket {
            key,
            entry_token,
            cycle: 1,
            value_version: 0,
            checkpoint_revision: 1,
            history_continuity: HistoryContinuity::NoCheckpoint,
            hint: None,
            previous_success: None,
            cleaned_prefix_len: None,
        })
    }

    #[cfg(test)]
    pub(crate) fn complete(
        &self,
        ticket: &HoldTicket,
        preference: HoldPreferenceV1,
        now: Instant,
    ) -> HoldCompleteOutcome {
        self.complete_with_cleanup(ticket, preference, None, now)
    }

    pub(crate) fn complete_with_cleanup(
        &self,
        ticket: &HoldTicket,
        preference: HoldPreferenceV1,
        cleaned_prefix_len: Option<usize>,
        now: Instant,
    ) -> HoldCompleteOutcome {
        let mut inner = self.inner.lock();
        inner.purge_expired_bounded(now, self.idle_ttl);
        let key = &ticket.key;
        inner.remove_if_expired(key, now, self.idle_ttl);
        let Some(entry) = inner.entries.get(key) else {
            return HoldCompleteOutcome::Stale;
        };
        if entry.entry_token != ticket.entry_token
            || entry.checkpoint_revision != ticket.checkpoint_revision
            || entry.cycle != ticket.cycle
            || entry.value_version != ticket.value_version
        {
            return HoldCompleteOutcome::Stale;
        }
        let Some(next_value_version) = entry.value_version.checked_add(1) else {
            return HoldCompleteOutcome::Stale;
        };
        if !inner.make_room_for_value(key, &preference, self.max_memory_bytes) {
            return HoldCompleteOutcome::CapacityMiss;
        }
        let Some(entry) = inner.entries.get_mut(key) else {
            return HoldCompleteOutcome::Stale;
        };
        if entry.entry_token != ticket.entry_token
            || entry.checkpoint_revision != ticket.checkpoint_revision
            || entry.cycle != ticket.cycle
            || entry.value_version != ticket.value_version
        {
            return HoldCompleteOutcome::Stale;
        }
        let previous_dynamic = entry.preference.as_ref().map_or(0, preference_bytes);
        let next_dynamic = preference_bytes(&preference);
        entry.preference = Some(preference);
        entry.hint_valid = true;
        entry.cleaned_prefix_len =
            cleaned_prefix_len.filter(|end| *end > 0 && *end <= entry.message_count);
        entry.value_version = next_value_version;
        inner.dynamic_bytes = inner
            .dynamic_bytes
            .saturating_sub(previous_dynamic)
            .saturating_add(next_dynamic);
        inner.peak_accounted_bytes = inner.peak_accounted_bytes.max(inner.accounted_bytes());
        HoldCompleteOutcome::Applied
    }

    #[cfg(test)]
    pub(crate) fn accounted_bytes(&self) -> usize {
        self.inner.lock().accounted_bytes()
    }

    #[cfg(test)]
    pub(crate) fn entry_count(&self) -> usize {
        self.inner.lock().entries.len()
    }

    #[cfg(test)]
    pub(crate) fn peak_accounted_bytes(&self) -> usize {
        self.inner.lock().peak_accounted_bytes
    }

    #[cfg(test)]
    pub(crate) fn maintenance_visits(&self) -> usize {
        self.inner.lock().maintenance_visits
    }
}

impl StoreInner {
    fn purge_expired_bounded(&mut self, now: Instant, idle_ttl: Duration) {
        for _ in 0..MAINTENANCE_BATCH {
            let Some(oldest) = self.lru.first().cloned() else {
                break;
            };
            #[cfg(test)]
            {
                self.maintenance_visits = self.maintenance_visits.saturating_add(1);
            }
            if !is_expired(oldest.last_access, now, idle_ttl) {
                break;
            }
            self.remove_entry(&oldest.key);
        }
    }

    fn remove_if_expired(
        &mut self,
        key: &ContextHoldKey,
        now: Instant,
        idle_ttl: Duration,
    ) -> bool {
        let expired = self
            .entries
            .get(key)
            .is_some_and(|entry| is_expired(entry.last_access, now, idle_ttl));
        expired && self.remove_entry(key)
    }

    fn make_room_for_insert(&mut self, entry: &HoldEntry, budget: usize) -> bool {
        let dynamic = entry.dynamic_bytes();
        for _ in 0..=EVICTION_BATCH {
            let projected = retained_bytes(
                self.entries.len().saturating_add(1),
                self.dynamic_bytes.saturating_add(dynamic),
            );
            if projected <= budget {
                self.peak_accounted_bytes = self.peak_accounted_bytes.max(projected);
                return true;
            }
            if !self.evict_lru(None) {
                return false;
            }
        }
        false
    }

    fn make_room_for_value(
        &mut self,
        protected: &ContextHoldKey,
        preference: &HoldPreferenceV1,
        budget: usize,
    ) -> bool {
        for _ in 0..=EVICTION_BATCH {
            let current = self
                .entries
                .get(protected)
                .map(HoldEntry::dynamic_bytes)
                .unwrap_or(0);
            let projected = self
                .accounted_bytes()
                .saturating_sub(current)
                .saturating_add(preference_bytes(preference));
            if projected <= budget {
                self.peak_accounted_bytes = self.peak_accounted_bytes.max(projected);
                return true;
            }
            if !self.evict_lru(Some(protected)) {
                return false;
            }
        }
        false
    }

    fn evict_lru(&mut self, protected: Option<&ContextHoldKey>) -> bool {
        let victim = self
            .lru
            .iter()
            .find(|record| protected.is_none_or(|protected| &record.key != protected))
            .map(|record| record.key.clone());
        victim.is_some_and(|victim| self.remove_entry(&victim))
    }

    fn accounted_bytes(&self) -> usize {
        retained_bytes(self.entries.len(), self.dynamic_bytes)
    }

    fn allocate_entry_token(&mut self) -> Option<u64> {
        let token = self.next_token;
        self.next_token = self.next_token.checked_add(1)?;
        Some(token)
    }

    fn allocate_access_order(&mut self) -> Option<u64> {
        self.access_clock = self.access_clock.checked_add(1)?;
        Some(self.access_clock)
    }

    fn touch(&mut self, key: &ContextHoldKey, now: Instant, access_order: u64) {
        let Some(entry) = self.entries.get_mut(key) else {
            return;
        };
        self.lru.remove(&LruRecord {
            last_access: entry.last_access,
            access_order: entry.access_order,
            key: key.clone(),
        });
        entry.last_access = now;
        entry.access_order = access_order;
        self.lru.insert(LruRecord {
            last_access: now,
            access_order,
            key: key.clone(),
        });
    }

    fn remove_entry(&mut self, key: &ContextHoldKey) -> bool {
        let Some(entry) = self.entries.remove(key) else {
            return false;
        };
        self.lru.remove(&LruRecord {
            last_access: entry.last_access,
            access_order: entry.access_order,
            key: key.clone(),
        });
        self.dynamic_bytes = self.dynamic_bytes.saturating_sub(entry.dynamic_bytes());
        true
    }
}

impl BeginSnapshot {
    fn matches_checkpoint(self, entry: &HoldEntry) -> bool {
        entry.entry_token == self.entry_token
            && entry.cycle == self.cycle
            && entry.checkpoint_revision == self.checkpoint_revision
            && entry.instruction_digest == self.instruction_digest
            && entry.message_count == self.message_count
            && entry.history_digest == self.history_digest
    }
}

impl From<&HoldEntry> for BeginSnapshot {
    fn from(entry: &HoldEntry) -> Self {
        Self {
            entry_token: entry.entry_token,
            cycle: entry.cycle,
            checkpoint_revision: entry.checkpoint_revision,
            instruction_digest: entry.instruction_digest,
            message_count: entry.message_count,
            history_digest: entry.history_digest,
        }
    }
}

impl HoldEntry {
    fn dynamic_bytes(&self) -> usize {
        self.preference.as_ref().map_or(0, preference_bytes)
    }
}

fn preference_bytes(preference: &HoldPreferenceV1) -> usize {
    preference.stable_binding_id.capacity()
        + preference.candidate_id.capacity()
        + preference.profile_digest.capacity()
        + preference.reasoning_profile_id.capacity()
        + preference.origin_group_id.capacity()
}

fn tree_index_bytes(len: usize) -> usize {
    if len == 0 {
        0
    } else {
        // Covers both tree indexes, allocator headers, partially occupied
        // nodes and a node-split peak without relying on private std growth.
        TREE_INDEX_BASE_BYTES.saturating_add(len.saturating_mul(TREE_INDEX_ENTRY_BYTES))
    }
}

fn store_fixed_bytes() -> usize {
    // ContextHoldStore includes the inline Mutex/StoreInner containers. The
    // additional allowance covers allocator headers and implementation slack.
    size_of::<ContextHoldStore>().saturating_add(512)
}

fn retained_bytes(len: usize, dynamic: usize) -> usize {
    store_fixed_bytes()
        .saturating_add(tree_index_bytes(len))
        .saturating_add(dynamic)
}

fn is_expired(last_access: Instant, now: Instant, idle_ttl: Duration) -> bool {
    now.checked_duration_since(last_access)
        .is_some_and(|idle| idle >= idle_ttl)
}

#[cfg(test)]
#[path = "store_tests.rs"]
mod tests;