Skip to content

Make Codex and Claude Code subscriptions go further: a local model router that gives routine agent work to economical models and keeps your strongest model for the hard parts.

15.8 KB
Raw Blame
use super::*;
use crate::server::core_runtime::model_ir::{
    CanonicalInstruction, CanonicalMessage, ContentPart, InstructionRole, MODEL_REQUEST_IR_SCHEMA,
    MessageRole, ModelRequestIRV1, RequestedReasoningControl, ToolChoice,
};
use crate::server::request_plan::IngressProtocol;

const HISTORY_KEY: [u8; 32] = [23; 32];

fn key(id: usize) -> ContextHoldKey {
    let mut digest = [0_u8; 32];
    let bytes = id.to_be_bytes();
    digest[..bytes.len()].copy_from_slice(&bytes);
    ContextHoldKey(digest)
}

fn history_request(instruction: u8, messages: &[u8]) -> ModelRequestIRV1 {
    ModelRequestIRV1 {
        native_body: None,
        native_only: false,
        schema_version: MODEL_REQUEST_IR_SCHEMA.into(),
        ingress_protocol: IngressProtocol::Responses,
        served_model_id: "agent/test".into(),
        stream: false,
        instructions: vec![CanonicalInstruction {
            role: InstructionRole::Developer,
            content: vec![ContentPart::Text {
                text: format!("instruction-{instruction}"),
            }],
        }],
        messages: messages
            .iter()
            .map(|message| CanonicalMessage {
                role: MessageRole::User,
                content: vec![ContentPart::Text {
                    text: format!("message-{message}"),
                }],
                name: None,
            })
            .collect(),
        tools: Vec::new(),
        tool_namespaces: Vec::new(),
        responses_tool_order: Vec::new(),
        web_search: None,
        responses_search_history: Default::default(),
        responses_annotations: Default::default(),
        responses_message_phases: Default::default(),
        responses_internal_chat_message_metadata: Default::default(),
        responses_reasoning_history: Default::default(),
        tool_choice: ToolChoice::None,
        parallel_tool_calls: false,
        requested_reasoning: RequestedReasoningControl::absent(),
        requested_max_output_tokens: None,
        provider_state: Vec::new(),
        responses_options: None,
        responses_item_ids: Default::default(),
        responses_item_statuses: Default::default(),
    }
}

fn begin(
    store: &ContextHoldStore,
    scope: usize,
    instruction: u8,
    messages: &[u8],
    now: Instant,
) -> Option<HoldTicket> {
    let request = history_request(instruction, messages);
    let history = super::super::visible_history(&request, &HISTORY_KEY)?;
    store.begin(key(scope), &history, now)
}

fn preference(id: &str) -> HoldPreferenceV1 {
    HoldPreferenceV1 {
        stable_binding_id: format!("binding-{id}"),
        candidate_id: id.into(),
        profile_digest: format!("sha256:{id}"),
        reasoning_profile_id: "fixed".into(),
        origin_group_id: "group".into(),
    }
}

#[test]
fn append_and_repeat_keep_while_rebuild_clears() {
    let store = ContextHoldStore::new(DEFAULT_MAX_MEMORY_BYTES, DEFAULT_IDLE_TTL);
    let now = Instant::now();
    let first = begin(&store, 1, 9, &[1], now).unwrap();
    assert_eq!(first.history_continuity, HistoryContinuity::NoCheckpoint);
    assert!(first.previous_success.is_none());
    assert_eq!(
        store.complete(&first, preference("a"), now),
        HoldCompleteOutcome::Applied
    );

    let appended = begin(&store, 1, 9, &[1, 2], now + Duration::from_secs(1)).unwrap();
    assert_eq!(appended.history_continuity, HistoryContinuity::Continuing);
    assert_eq!(appended.hint.as_ref().unwrap().candidate_id, "a");
    assert_eq!(
        appended.previous_success.as_ref().unwrap().candidate_id,
        "a"
    );
    let repeated = begin(&store, 1, 9, &[1, 2], now + Duration::from_secs(2)).unwrap();
    assert_eq!(repeated.history_continuity, HistoryContinuity::Continuing);
    assert_eq!(repeated.hint.as_ref().unwrap().candidate_id, "a");
    let changed_instructions = begin(&store, 1, 10, &[1, 2], now + Duration::from_secs(3)).unwrap();
    assert_eq!(
        changed_instructions.history_continuity,
        HistoryContinuity::Continuing
    );
    assert!(changed_instructions.hint.is_none());
    assert_eq!(
        changed_instructions.previous_success.unwrap().candidate_id,
        "a"
    );
    let rebuilt = begin(&store, 1, 10, &[1, 3], now + Duration::from_secs(4)).unwrap();
    assert_eq!(rebuilt.history_continuity, HistoryContinuity::Rewritten);
    assert!(rebuilt.hint.is_none());
    assert_eq!(rebuilt.previous_success.unwrap().candidate_id, "a");
}

#[test]
fn rebuilt_history_exposes_only_prior_complete_delivery_for_fallback() {
    let store = ContextHoldStore::new(DEFAULT_MAX_MEMORY_BYTES, DEFAULT_IDLE_TTL);
    let now = Instant::now();
    let first = begin(&store, 1, 9, &[1], now).unwrap();
    let without_success = begin(&store, 1, 9, &[2], now + Duration::from_secs(1)).unwrap();
    assert!(without_success.previous_success.is_none());
    assert_eq!(
        store.complete(&without_success, preference("luna"), now),
        HoldCompleteOutcome::Applied
    );
    let compacted = begin(&store, 1, 9, &[3], now + Duration::from_secs(2)).unwrap();
    assert_eq!(compacted.history_continuity, HistoryContinuity::Rewritten);
    assert!(compacted.hint.is_none());
    assert_eq!(compacted.previous_success.unwrap().candidate_id, "luna");
    assert_eq!(
        store.complete(&first, preference("stale"), now),
        HoldCompleteOutcome::Stale
    );
}

#[test]
fn cleaned_prefix_stays_fixed_and_does_not_restore_invalidated_hint() {
    let store = ContextHoldStore::new(DEFAULT_MAX_MEMORY_BYTES, DEFAULT_IDLE_TTL);
    let now = Instant::now();
    let first = begin(&store, 1, 9, &[1, 2], now).unwrap();
    assert_eq!(
        store.complete_with_cleanup(&first, preference("b"), Some(2), now),
        HoldCompleteOutcome::Applied
    );
    let appended = begin(&store, 1, 9, &[1, 2, 3], now).unwrap();
    assert_eq!(appended.cleaned_prefix_len, Some(2));
    assert_eq!(appended.hint.as_ref().unwrap().candidate_id, "b");
    assert_eq!(
        store.complete_with_cleanup(&appended, preference("b"), appended.cleaned_prefix_len, now),
        HoldCompleteOutcome::Applied
    );
    let changed_tools = begin(&store, 1, 10, &[1, 2, 3, 4], now).unwrap();
    assert_eq!(changed_tools.cleaned_prefix_len, Some(2));
    assert!(changed_tools.hint.is_none());
    let repeated = begin(&store, 1, 10, &[1, 2, 3, 4], now).unwrap();
    assert_eq!(repeated.cleaned_prefix_len, Some(2));
    assert!(repeated.hint.is_none());
    assert_eq!(
        store.complete_with_cleanup(&changed_tools, preference("a"), Some(4), now),
        HoldCompleteOutcome::Stale
    );
    let compacted = begin(&store, 1, 10, &[8], now).unwrap();
    assert_eq!(compacted.cleaned_prefix_len, None);
    assert_eq!(
        store.complete_with_cleanup(&repeated, preference("b"), Some(2), now),
        HoldCompleteOutcome::Stale
    );
    assert_eq!(
        begin(&store, 2, 10, &[1, 2, 3, 4], now)
            .unwrap()
            .cleaned_prefix_len,
        None
    );
}

#[test]
fn latest_history_checkpoint_fences_repeat_and_append_completions() {
    let store = ContextHoldStore::new(DEFAULT_MAX_MEMORY_BYTES, DEFAULT_IDLE_TTL);
    let now = Instant::now();
    let first = begin(&store, 1, 9, &[1], now).unwrap();
    let repeated = begin(&store, 1, 9, &[1], now + Duration::from_millis(1)).unwrap();
    assert_eq!(
        store.complete(&first, preference("a"), now),
        HoldCompleteOutcome::Stale
    );
    assert_eq!(
        store.complete(&repeated, preference("b"), now),
        HoldCompleteOutcome::Applied
    );
    assert_eq!(
        begin(&store, 1, 9, &[1, 2], now + Duration::from_secs(1))
            .unwrap()
            .hint
            .unwrap()
            .candidate_id,
        "b"
    );

    let earlier = begin(&store, 2, 9, &[1], now).unwrap();
    let appended = begin(&store, 2, 9, &[1, 2], now + Duration::from_millis(1)).unwrap();
    assert_eq!(
        store.complete(&earlier, preference("earlier"), now),
        HoldCompleteOutcome::Stale
    );
    assert_eq!(
        store.complete(&appended, preference("appended"), now),
        HoldCompleteOutcome::Applied
    );
    assert_eq!(
        begin(&store, 2, 9, &[1, 2, 3], now + Duration::from_secs(1))
            .unwrap()
            .hint
            .unwrap()
            .candidate_id,
        "appended"
    );

    let slow = begin(&store, 3, 9, &[1], now).unwrap();
    let fast = begin(&store, 3, 9, &[1, 2], now + Duration::from_millis(1)).unwrap();
    assert_eq!(
        store.complete(&fast, preference("fast"), now),
        HoldCompleteOutcome::Applied
    );
    assert_eq!(
        store.complete(&slow, preference("slow"), now),
        HoldCompleteOutcome::Stale
    );
    assert_eq!(
        begin(&store, 3, 9, &[1, 2, 3], now + Duration::from_secs(1))
            .unwrap()
            .hint
            .unwrap()
            .candidate_id,
        "fast"
    );
}

#[test]
fn completion_value_version_is_independent_from_history_checkpoint_revision() {
    let store = ContextHoldStore::new(DEFAULT_MAX_MEMORY_BYTES, DEFAULT_IDLE_TTL);
    let now = Instant::now();
    let ticket = begin(&store, 1, 9, &[1], now).unwrap();
    let snapshot = {
        let inner = store.inner.lock();
        BeginSnapshot::from(inner.entries.get(&ticket.key).unwrap())
    };

    assert_eq!(
        store.complete(&ticket, preference("winner"), now),
        HoldCompleteOutcome::Applied
    );
    let inner = store.inner.lock();
    let completed = inner.entries.get(&ticket.key).unwrap();
    assert!(snapshot.matches_checkpoint(completed));
    assert_ne!(completed.value_version, ticket.value_version);
}

#[test]
fn rebuilt_cycle_rejects_prior_completion_without_rewinding_checkpoint() {
    let store = ContextHoldStore::new(DEFAULT_MAX_MEMORY_BYTES, DEFAULT_IDLE_TTL);
    let now = Instant::now();
    let old = begin(&store, 1, 9, &[1], now).unwrap();
    let rebuilt = begin(&store, 1, 9, &[2], now + Duration::from_millis(1)).unwrap();
    assert_eq!(
        store.complete(&old, preference("old"), now),
        HoldCompleteOutcome::Stale
    );
    assert_eq!(
        store.complete(&rebuilt, preference("new"), now),
        HoldCompleteOutcome::Applied
    );
    assert_eq!(
        begin(&store, 1, 9, &[2, 3], now + Duration::from_secs(1))
            .unwrap()
            .hint
            .unwrap()
            .candidate_id,
        "new"
    );
}

#[test]
fn idle_expiry_and_capacity_never_reinsert_old_completion() {
    let now = Instant::now();
    let store = ContextHoldStore::new(DEFAULT_MAX_MEMORY_BYTES, Duration::from_secs(60));
    let expired = begin(&store, 1, 9, &[1], now).unwrap();
    let after_expiry = now + Duration::from_secs(60);
    let replacement = begin(&store, 1, 9, &[1], after_expiry).unwrap();
    assert_eq!(
        replacement.history_continuity,
        HistoryContinuity::NoCheckpoint
    );
    assert_eq!(
        store.complete(&expired, preference("old"), after_expiry),
        HoldCompleteOutcome::Stale
    );
    assert_eq!(
        store.complete(&replacement, preference("new"), after_expiry),
        HoldCompleteOutcome::Applied
    );

    let tiny_budget = store_fixed_bytes().saturating_add(1);
    let tiny = ContextHoldStore::new(tiny_budget, DEFAULT_IDLE_TTL);
    assert!(begin(&tiny, 1, 9, &[1], now).is_none());
    assert!(tiny.accounted_bytes() <= tiny_budget);

    let eviction_budget = retained_bytes(3, 0);
    let evicting = ContextHoldStore::new(eviction_budget, DEFAULT_IDLE_TTL);
    let evicted = begin(&evicting, 10, 9, &[1], now).unwrap();
    assert!(begin(&evicting, 11, 9, &[1], now).is_some());
    assert!(begin(&evicting, 12, 9, &[1], now).is_some());
    assert!(begin(&evicting, 13, 9, &[1], now).is_some());
    assert_eq!(
        evicting.complete(&evicted, preference("evicted"), now),
        HoldCompleteOutcome::Stale
    );
    assert_eq!(
        begin(&evicting, 10, 9, &[9], now)
            .unwrap()
            .history_continuity,
        HistoryContinuity::NoCheckpoint
    );
    assert!(evicting.peak_accounted_bytes() <= eviction_budget);
}

#[test]
fn byte_budget_has_no_4096_entry_ceiling() {
    let store = ContextHoldStore::new(DEFAULT_MAX_MEMORY_BYTES, DEFAULT_IDLE_TTL);
    let now = Instant::now();
    for id in 0..4_200 {
        assert!(begin(&store, id, 9, &[1], now).is_some());
    }
    assert!(store.entry_count() > 4_096);
    assert!(store.accounted_bytes() <= DEFAULT_MAX_MEMORY_BYTES);
    assert!(store.peak_accounted_bytes() <= DEFAULT_MAX_MEMORY_BYTES);
}

#[test]
fn active_entries_have_no_absolute_ttl_and_maintenance_is_bounded() {
    let now = Instant::now();
    let active = ContextHoldStore::new(DEFAULT_MAX_MEMORY_BYTES, DEFAULT_IDLE_TTL);
    let first = begin(&active, 1, 9, &[1], now).unwrap();
    assert_eq!(
        active.complete(&first, preference("kept"), now),
        HoldCompleteOutcome::Applied
    );
    for hour in 1..=3 {
        let ticket = begin(
            &active,
            1,
            9,
            &[1],
            now + Duration::from_secs(hour * 59 * 60),
        )
        .unwrap();
        assert_eq!(ticket.hint.unwrap().candidate_id, "kept");
    }

    let expired = ContextHoldStore::new(DEFAULT_MAX_MEMORY_BYTES, Duration::from_secs(1));
    for id in 0..100 {
        assert!(begin(&expired, id, 9, &[1], now).is_some());
    }
    let before = expired.maintenance_visits();
    assert!(begin(&expired, 101, 9, &[1], now + Duration::from_secs(1)).is_some());
    let visits = expired.maintenance_visits().saturating_sub(before);
    assert!(visits <= MAINTENANCE_BATCH);
    assert!(expired.entry_count() >= 100 - MAINTENANCE_BATCH);
}

#[test]
fn ordered_indexes_charge_each_entry_before_insertion() {
    assert!(retained_bytes(4, 0) > retained_bytes(3, 0));
    assert_eq!(
        retained_bytes(4, 0).saturating_sub(retained_bytes(3, 0)),
        TREE_INDEX_ENTRY_BYTES
    );
}

#[test]
fn history_rewrites_and_unavailable_checkpoints_have_distinct_causes() {
    let now = Instant::now();
    for (replacement, expected) in [
        (&[9][..], HistoryContinuity::Shortened),
        (&[1, 2][..], HistoryContinuity::Shortened),
        (&[1, 9, 3][..], HistoryContinuity::Rewritten),
        (&[1, 9, 3, 4][..], HistoryContinuity::Rewritten),
    ] {
        let store = ContextHoldStore::new(DEFAULT_MAX_MEMORY_BYTES, DEFAULT_IDLE_TTL);
        assert_eq!(
            begin(&store, 1, 1, &[1, 2, 3], now)
                .unwrap()
                .history_continuity,
            HistoryContinuity::NoCheckpoint
        );
        // Only a decrease is the compaction heuristic, including a shorter rewind.
        assert_eq!(
            begin(&store, 1, 1, replacement, now)
                .unwrap()
                .history_continuity,
            expected
        );
    }
    let store = ContextHoldStore::new(DEFAULT_MAX_MEMORY_BYTES, Duration::from_secs(1));
    begin(&store, 1, 1, &[1, 2, 3], now).unwrap();
    assert_eq!(
        begin(&store, 1, 1, &[9], now + Duration::from_secs(1))
            .unwrap()
            .history_continuity,
        HistoryContinuity::NoCheckpoint
    );
    let disabled = ContextHoldStore {
        key: None,
        ..ContextHoldStore::default()
    };
    assert!(begin(&disabled, 1, 1, &[9], now).is_none());
}

#[test]
fn checkpoint_cas_contention_does_not_claim_a_rewrite() {
    let store = ContextHoldStore::default();
    let now = Instant::now();
    begin(&store, 1, 1, &[1], now).unwrap();
    let request = history_request(1, &[9]);
    let history = super::super::visible_history(&request, &HISTORY_KEY).unwrap();
    let contested = store.begin_with_measure(key(1), &history, now, |prefix| {
        // Deterministically interleave another entry while the original hashes outside the lock.
        assert_eq!(
            begin(&store, 1, 1, &[1, 2], now)
                .unwrap()
                .history_continuity,
            HistoryContinuity::Continuing
        );
        history.measure(prefix)
    });
    assert!(contested.is_none());
    assert_eq!(
        begin(&store, 1, 1, &[1, 2, 3], now)
            .unwrap()
            .history_continuity,
        HistoryContinuity::Continuing
    );
}