Skip to content

Make Codex and Claude Code subscriptions go further: a local model router that gives routine agent work to economical models and keeps your strongest model for the hard parts.

12.1 KB
Raw Blame
use std::collections::BTreeMap;
use std::sync::Arc;
use std::time::{Duration, Instant};

use async_trait::async_trait;
use parking_lot::Mutex;
use tokio_util::sync::CancellationToken;

use crate::attempt_outcome::{AttemptFailure, AttemptFailureClass};
use crate::ports::{
    CasOutcome, ExecutionScope, InMemoryRuntimeStateStore, ProbeLeaseOutcome, RuntimeHealth,
    RuntimeStateEntry, RuntimeStateError, RuntimeStateKey, RuntimeStateStore,
};

use super::state::{
    OwnedAttemptStatePermits, RuntimeCooldownPolicy, acquire_target_permit, confirm_success,
    record_failure, record_failure_at, validate_permit,
};

#[derive(Default)]
struct IndependentCasStore {
    entries: Mutex<BTreeMap<RuntimeStateKey, RuntimeStateEntry>>,
}

#[async_trait]
impl RuntimeStateStore for IndependentCasStore {
    async fn read(
        &self,
        key: &RuntimeStateKey,
        _scope: &ExecutionScope,
    ) -> Result<RuntimeStateEntry, RuntimeStateError> {
        Ok(self.entries.lock().get(key).cloned().unwrap_or_default())
    }

    async fn compare_and_swap(
        &self,
        key: &RuntimeStateKey,
        expected_generation: u64,
        next: RuntimeStateEntry,
        _scope: &ExecutionScope,
    ) -> Result<CasOutcome, RuntimeStateError> {
        let required_generation = expected_generation
            .checked_add(1)
            .ok_or(RuntimeStateError::Rejected)?;
        let mut entries = self.entries.lock();
        let current = entries.get(key).cloned().unwrap_or_default();
        if current.generation != expected_generation || next.generation != required_generation {
            return Ok(CasOutcome::Conflict);
        }
        entries.insert(key.clone(), next);
        Ok(CasOutcome::Applied {
            generation: required_generation,
        })
    }

    async fn acquire_probe_lease(
        &self,
        _key: &RuntimeStateKey,
        _expected_generation: u64,
        _now: Instant,
        _lease_duration: Duration,
        _scope: &ExecutionScope,
    ) -> Result<ProbeLeaseOutcome, RuntimeStateError> {
        Err(RuntimeStateError::Unavailable)
    }
}

fn scope() -> ExecutionScope {
    ExecutionScope::new(
        Instant::now() + Duration::from_secs(2),
        CancellationToken::new(),
    )
}

#[tokio::test]
async fn runtime_state_independent_adapter_allows_only_one_same_generation_writer() {
    let store = Arc::new(IndependentCasStore::default());
    let key = RuntimeStateKey::binding("binding-a");
    let disabled = RuntimeStateEntry {
        generation: 1,
        health: RuntimeHealth::Disabled,
        probe_lease_until: None,
        transient_backoff_step: 0,
    };
    let cooling = RuntimeStateEntry {
        generation: 1,
        health: RuntimeHealth::CoolingDown {
            until: Instant::now() + Duration::from_secs(30),
        },
        probe_lease_until: None,
        transient_backoff_step: 0,
    };
    let left = {
        let store = Arc::clone(&store);
        let key = key.clone();
        let scope = scope();
        tokio::spawn(async move { store.compare_and_swap(&key, 0, disabled, &scope).await })
    };
    let right = {
        let store = Arc::clone(&store);
        let key = key.clone();
        let scope = scope();
        tokio::spawn(async move { store.compare_and_swap(&key, 0, cooling, &scope).await })
    };
    let outcomes = [left.await.unwrap().unwrap(), right.await.unwrap().unwrap()];
    assert_eq!(
        outcomes
            .iter()
            .filter(|outcome| matches!(outcome, CasOutcome::Applied { generation: 1 }))
            .count(),
        1
    );
    assert_eq!(
        outcomes
            .iter()
            .filter(|outcome| **outcome == CasOutcome::Conflict)
            .count(),
        1
    );
    assert_eq!(store.read(&key, &scope()).await.unwrap().generation, 1);
}

#[tokio::test]
async fn runtime_state_in_memory_adapter_rejects_non_incrementing_next_generation() {
    let store = InMemoryRuntimeStateStore::default();
    let key = RuntimeStateKey::binding("binding-a");
    let outcome = store
        .compare_and_swap(
            &key,
            0,
            RuntimeStateEntry {
                generation: 0,
                health: RuntimeHealth::Disabled,
                probe_lease_until: None,
                transient_backoff_step: 0,
            },
            &scope(),
        )
        .await
        .unwrap();
    assert_eq!(outcome, CasOutcome::Conflict);
    assert_eq!(store.entry(&key), RuntimeStateEntry::default());
}

#[tokio::test]
async fn expired_cooldown_probe_lease_covers_the_bounded_attempt() {
    let store = InMemoryRuntimeStateStore::default();
    let key = RuntimeStateKey::binding("slow-model-binding");
    store.insert(
        key.clone(),
        RuntimeStateEntry {
            generation: 7,
            health: RuntimeHealth::CoolingDown {
                until: Instant::now() - Duration::from_secs(1),
            },
            probe_lease_until: None,
            transient_backoff_step: 0,
        },
    );
    let started = Instant::now();
    let deadline = started + Duration::from_secs(30);
    let scope = ExecutionScope::new(deadline, CancellationToken::new());

    let permit = acquire_target_permit(&store, &key, &scope)
        .await
        .unwrap()
        .expect("expired cooldown must elect one probe");
    let lease_until = store
        .entry(&key)
        .probe_lease_until
        .expect("probe lease must be persisted");

    assert!(lease_until > started + Duration::from_secs(25));
    assert!(lease_until <= deadline);
    validate_permit(&store, &key, permit, &scope)
        .await
        .expect("a slow but in-deadline attempt must keep its probe authority");
}

#[tokio::test]
async fn credential_scoped_failure_without_a_credential_cools_only_the_binding() {
    let store = InMemoryRuntimeStateStore::default();
    let key = RuntimeStateKey::binding("no-credential-binding");
    let request_scope = scope();
    let permit = acquire_target_permit(&store, &key, &request_scope)
        .await
        .unwrap()
        .unwrap();
    let permits = OwnedAttemptStatePermits::without_credential(key.clone(), permit);
    record_failure(
        &store,
        permits.borrowed(),
        &AttemptFailure {
            class: AttemptFailureClass::Quota,
            status: Some(429),
            retry_after: Some(Duration::from_secs(5)),
        },
        &RuntimeCooldownPolicy::default(),
        &request_scope,
    )
    .await
    .unwrap();

    assert!(matches!(
        store.entry(&key).health,
        RuntimeHealth::CoolingDown { .. }
    ));
    assert_eq!(store.entry(&key).transient_backoff_step, 0);
}

#[tokio::test]
async fn binding_transient_backoff_saturates_and_trusted_retry_after_resets_it() {
    let store = InMemoryRuntimeStateStore::default();
    let key = RuntimeStateKey::binding("binding-backoff");
    let request_scope = scope();
    let failure = AttemptFailure {
        class: AttemptFailureClass::Transient,
        status: Some(503),
        retry_after: None,
    };

    for (expected_step, expected_seconds) in [(1, 2), (2, 4), (3, 8), (4, 16), (5, 30), (5, 30)] {
        let permit = acquire_target_permit(&store, &key, &request_scope)
            .await
            .unwrap()
            .unwrap();
        let permits = OwnedAttemptStatePermits::without_credential(key.clone(), permit);
        let now = Instant::now();
        record_failure_at(
            &store,
            permits.borrowed(),
            &failure,
            &RuntimeCooldownPolicy::default(),
            &request_scope,
            now,
        )
        .await
        .unwrap();
        let mut state = store.entry(&key);
        assert_eq!(state.transient_backoff_step, expected_step);
        assert_eq!(
            state.health,
            RuntimeHealth::CoolingDown {
                until: now + Duration::from_secs(expected_seconds)
            }
        );
        state.health = RuntimeHealth::CoolingDown {
            until: Instant::now() - Duration::from_millis(1),
        };
        state.probe_lease_until = None;
        store.insert(key.clone(), state);
    }

    let permit = acquire_target_permit(&store, &key, &request_scope)
        .await
        .unwrap()
        .unwrap();
    let permits = OwnedAttemptStatePermits::without_credential(key.clone(), permit);
    let now = Instant::now();
    record_failure_at(
        &store,
        permits.borrowed(),
        &AttemptFailure {
            class: AttemptFailureClass::Transient,
            status: Some(503),
            retry_after: Some(Duration::from_secs(77)),
        },
        &RuntimeCooldownPolicy::default(),
        &request_scope,
        now,
    )
    .await
    .unwrap();
    let state = store.entry(&key);
    assert_eq!(state.transient_backoff_step, 0);
    assert_eq!(
        state.health,
        RuntimeHealth::CoolingDown {
            until: now + Duration::from_secs(77)
        }
    );
}

#[tokio::test]
async fn stale_binding_failure_cannot_increment_the_same_backoff_state_twice() {
    let store = InMemoryRuntimeStateStore::default();
    let key = RuntimeStateKey::binding("binding-concurrent-backoff");
    let request_scope = scope();
    let left = acquire_target_permit(&store, &key, &request_scope)
        .await
        .unwrap()
        .unwrap();
    let right = acquire_target_permit(&store, &key, &request_scope)
        .await
        .unwrap()
        .unwrap();
    let failure = AttemptFailure {
        class: AttemptFailureClass::Transient,
        status: Some(503),
        retry_after: None,
    };
    let now = Instant::now();
    let left = OwnedAttemptStatePermits::without_credential(key.clone(), left);
    record_failure_at(
        &store,
        left.borrowed(),
        &failure,
        &RuntimeCooldownPolicy::default(),
        &request_scope,
        now,
    )
    .await
    .unwrap();
    let right = OwnedAttemptStatePermits::without_credential(key.clone(), right);
    assert!(
        record_failure_at(
            &store,
            right.borrowed(),
            &failure,
            &RuntimeCooldownPolicy::default(),
            &request_scope,
            now,
        )
        .await
        .is_err()
    );
    let current = store.entry(&key);
    assert_eq!(current.generation, 1);
    assert_eq!(current.transient_backoff_step, 1);
}

#[tokio::test]
async fn overload_uses_default_and_successful_probe_resets_transient_step() {
    let store = InMemoryRuntimeStateStore::default();
    let key = RuntimeStateKey::binding("binding-overload");
    let request_scope = scope();
    store.insert(
        key.clone(),
        RuntimeStateEntry {
            generation: 4,
            health: RuntimeHealth::CoolingDown {
                until: Instant::now() - Duration::from_millis(1),
            },
            probe_lease_until: None,
            transient_backoff_step: 5,
        },
    );
    let permit = acquire_target_permit(&store, &key, &request_scope)
        .await
        .unwrap()
        .unwrap();
    let permits = OwnedAttemptStatePermits::without_credential(key.clone(), permit);
    let now = Instant::now();
    record_failure_at(
        &store,
        permits.borrowed(),
        &AttemptFailure {
            class: AttemptFailureClass::BindingOverload,
            status: Some(529),
            retry_after: None,
        },
        &RuntimeCooldownPolicy::default(),
        &request_scope,
        now,
    )
    .await
    .unwrap();
    let mut state = store.entry(&key);
    assert_eq!(state.transient_backoff_step, 0);
    assert_eq!(
        state.health,
        RuntimeHealth::CoolingDown {
            until: now + Duration::from_secs(15)
        }
    );

    state.health = RuntimeHealth::CoolingDown {
        until: Instant::now() - Duration::from_millis(1),
    };
    state.transient_backoff_step = 4;
    store.insert(key.clone(), state);
    let permit = acquire_target_permit(&store, &key, &request_scope)
        .await
        .unwrap()
        .unwrap();
    let permits = OwnedAttemptStatePermits::without_credential(key.clone(), permit);
    confirm_success(&store, permits.borrowed(), &request_scope)
        .await
        .unwrap();
    let active = store.entry(&key);
    assert_eq!(active.health, RuntimeHealth::Active);
    assert_eq!(active.transient_backoff_step, 0);
}