Make Codex and Claude Code subscriptions go further: a local model router that gives routine agent work to economical models and keeps your strongest model for the hard parts.
Rust ai-agentsai-gatewayclaude-codecodexcoding-agentcost-optimizationdeepseekllm-gatewayllm-routerlocal-firstmodel-routingrusttask-routingtauri
use super::*;
#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)]
#[repr(u8)]
pub enum MemoryRole {
RawRequest = 0,
ModelIrBacking = 1,
AttemptWire = 2,
Retry = 3,
ResponsePrefix = 4,
SseFrame = 5,
SemanticState = 6,
OutputQueue = 7,
TransportInflight = 8,
}
impl MemoryRole {
pub const ALL: [Self; 9] = [
Self::RawRequest,
Self::ModelIrBacking,
Self::AttemptWire,
Self::Retry,
Self::ResponsePrefix,
Self::SseFrame,
Self::SemanticState,
Self::OutputQueue,
Self::TransportInflight,
];
}
#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)]
pub struct StreamBudgetId(u64);
#[derive(Clone, Debug)]
pub struct BudgetTree {
inner: Arc<Mutex<BudgetState>>,
}
#[derive(Debug)]
struct BudgetState {
process_limit: usize,
worker_limit: usize,
process_live: usize,
worker_live: usize,
process_peak: usize,
worker_peak: usize,
process_rejected: usize,
worker_rejected: usize,
process_role_live: [usize; 9],
process_role_peak: [usize; 9],
worker_role_live: [usize; 9],
worker_role_peak: [usize; 9],
next_stream_id: u64,
streams: HashMap<StreamBudgetId, StreamState>,
}
#[derive(Debug)]
struct StreamState {
limit: usize,
live: usize,
peak: usize,
rejected: usize,
role_live: [usize; 9],
role_peak: [usize; 9],
}
impl BudgetTree {
pub fn new(process_limit: usize, worker_limit: usize) -> Result<Self, BodyError> {
if process_limit == 0 || worker_limit == 0 || worker_limit > process_limit {
return Err(BodyError::InvalidBudgetLimit);
}
Ok(Self {
inner: Arc::new(Mutex::new(BudgetState {
process_limit,
worker_limit,
process_live: 0,
worker_live: 0,
process_peak: 0,
worker_peak: 0,
process_rejected: 0,
worker_rejected: 0,
process_role_live: [0; 9],
process_role_peak: [0; 9],
worker_role_live: [0; 9],
worker_role_peak: [0; 9],
next_stream_id: 0,
streams: HashMap::new(),
})),
})
}
pub fn stream(&self, limit: usize) -> Result<StreamBudget, BodyError> {
self.stream_inner(limit, None)
}
pub fn stream_with_telemetry(
&self,
limit: usize,
telemetry: RequestTelemetry,
) -> Result<StreamBudget, BodyError> {
self.stream_inner(limit, Some(telemetry))
}
fn stream_inner(
&self,
limit: usize,
telemetry: Option<RequestTelemetry>,
) -> Result<StreamBudget, BodyError> {
let mut inner = lock(&self.inner);
if limit == 0 || limit > inner.worker_limit {
return Err(BodyError::InvalidBudgetLimit);
}
inner.next_stream_id = inner.next_stream_id.wrapping_add(1);
let id = StreamBudgetId(inner.next_stream_id);
inner.streams.insert(
id,
StreamState {
limit,
live: 0,
peak: 0,
rejected: 0,
role_live: [0; 9],
role_peak: [0; 9],
},
);
Ok(StreamBudget {
handle: Arc::new(StreamBudgetHandle {
id,
inner: Arc::clone(&self.inner),
telemetry,
}),
})
}
pub fn snapshot(&self) -> BudgetTreeSnapshot {
let inner = lock(&self.inner);
BudgetTreeSnapshot {
process_live: inner.process_live,
process_peak: inner.process_peak,
worker_live: inner.worker_live,
worker_peak: inner.worker_peak,
active_streams: inner.streams.len(),
}
}
pub(crate) fn owns_stream(&self, budget: &StreamBudget) -> bool {
Arc::ptr_eq(&self.inner, &budget.handle.inner)
}
}
#[derive(Clone, Debug)]
pub struct StreamBudget {
handle: Arc<StreamBudgetHandle>,
}
#[derive(Debug)]
struct StreamBudgetHandle {
id: StreamBudgetId,
inner: Arc<Mutex<BudgetState>>,
telemetry: Option<RequestTelemetry>,
}
impl Drop for StreamBudgetHandle {
fn drop(&mut self) {
let mut inner = lock(&self.inner);
if inner
.streams
.get(&self.id)
.is_some_and(|stream| stream.live == 0)
{
inner.streams.remove(&self.id);
}
}
}
impl StreamBudget {
pub fn reserve(&self, role: MemoryRole, bytes: usize) -> Result<Reservation, BodyError> {
if bytes == 0 {
return Ok(Reservation {
stream: self.clone(),
role,
bytes,
released: false,
});
}
let mut inner = lock(&self.handle.inner);
let process_next = inner
.process_live
.checked_add(bytes)
.ok_or(BodyError::BudgetExceeded)?;
let worker_next = inner
.worker_live
.checked_add(bytes)
.ok_or(BodyError::BudgetExceeded)?;
let (stream_next, role_next, stream_limit) = {
let stream = inner
.streams
.get(&self.handle.id)
.ok_or(BodyError::UnknownStreamBudget)?;
(
stream
.live
.checked_add(bytes)
.ok_or(BodyError::BudgetExceeded)?,
stream.role_live[role as usize]
.checked_add(bytes)
.ok_or(BodyError::BudgetExceeded)?,
stream.limit,
)
};
if process_next > inner.process_limit
|| worker_next > inner.worker_limit
|| stream_next > stream_limit
{
inner.process_rejected += 1;
inner.worker_rejected += 1;
inner
.streams
.get_mut(&self.handle.id)
.ok_or(BodyError::UnknownStreamBudget)?
.rejected += 1;
let snapshot = snapshot_for_observation(&inner, self.handle.id)?;
drop(inner);
self.observe_memory(role, snapshot);
return Err(BodyError::BudgetExceeded);
}
inner.process_live = process_next;
inner.worker_live = worker_next;
inner.process_peak = inner.process_peak.max(process_next);
inner.worker_peak = inner.worker_peak.max(worker_next);
inner.process_role_live[role as usize] = inner.process_role_live[role as usize]
.checked_add(bytes)
.ok_or(BodyError::BudgetExceeded)?;
inner.worker_role_live[role as usize] = inner.worker_role_live[role as usize]
.checked_add(bytes)
.ok_or(BodyError::BudgetExceeded)?;
inner.process_role_peak[role as usize] =
inner.process_role_peak[role as usize].max(inner.process_role_live[role as usize]);
inner.worker_role_peak[role as usize] =
inner.worker_role_peak[role as usize].max(inner.worker_role_live[role as usize]);
let stream = inner
.streams
.get_mut(&self.handle.id)
.ok_or(BodyError::UnknownStreamBudget)?;
stream.live = stream_next;
stream.peak = stream.peak.max(stream_next);
stream.role_live[role as usize] = role_next;
stream.role_peak[role as usize] = stream.role_peak[role as usize].max(role_next);
let snapshot = snapshot_for_observation(&inner, self.handle.id)?;
drop(inner);
self.observe_memory(role, snapshot);
Ok(Reservation {
stream: self.clone(),
role,
bytes,
released: false,
})
}
pub fn snapshot(&self) -> Result<StreamBudgetSnapshot, BodyError> {
let inner = lock(&self.handle.inner);
let stream = inner
.streams
.get(&self.handle.id)
.ok_or(BodyError::UnknownStreamBudget)?;
Ok(snapshot_from_stream(stream))
}
fn observe_memory(&self, role: MemoryRole, snapshot: BudgetMemorySnapshot) {
if let Some(telemetry) = &self.handle.telemetry {
telemetry.memory_role(role, snapshot);
}
}
}
#[derive(Debug)]
pub struct Reservation {
stream: StreamBudget,
role: MemoryRole,
bytes: usize,
released: bool,
}
/// Cloneable, opaque ownership of already-admitted queue metadata. The
/// reservation is shared only among units allocated from the same fixed
/// output queue and is released with the final downstream unit/drop.
#[derive(Clone, Debug, Default)]
pub struct BodyMetadataOwner {
reservation: Option<Arc<Reservation>>,
}
impl BodyMetadataOwner {
pub(crate) fn from_reservation(reservation: Option<Arc<Reservation>>) -> Self {
Self { reservation }
}
pub fn is_empty(&self) -> bool {
self.reservation.is_none()
}
}
impl Reservation {
pub fn bytes(&self) -> usize {
self.bytes
}
pub fn role(&self) -> MemoryRole {
self.role
}
pub fn transfer_role(&mut self, role: MemoryRole) -> Result<(), BodyError> {
if self.released {
return Err(BodyError::ReservationReleased);
}
if self.role == role || self.bytes == 0 {
self.role = role;
return Ok(());
}
let mut inner = lock(&self.stream.handle.inner);
inner.process_role_live[self.role as usize] -= self.bytes;
inner.worker_role_live[self.role as usize] -= self.bytes;
inner.process_role_live[role as usize] += self.bytes;
inner.worker_role_live[role as usize] += self.bytes;
inner.process_role_peak[role as usize] =
inner.process_role_peak[role as usize].max(inner.process_role_live[role as usize]);
inner.worker_role_peak[role as usize] =
inner.worker_role_peak[role as usize].max(inner.worker_role_live[role as usize]);
let stream = inner
.streams
.get_mut(&self.stream.handle.id)
.ok_or(BodyError::UnknownStreamBudget)?;
stream.role_live[self.role as usize] -= self.bytes;
let next = stream.role_live[role as usize]
.checked_add(self.bytes)
.ok_or(BodyError::BudgetExceeded)?;
stream.role_live[role as usize] = next;
stream.role_peak[role as usize] = stream.role_peak[role as usize].max(next);
let snapshot = snapshot_for_observation(&inner, self.stream.handle.id)?;
let old_role = self.role;
self.role = role;
drop(inner);
self.stream.observe_memory(old_role, snapshot);
self.stream.observe_memory(role, snapshot);
Ok(())
}
fn release(&mut self) {
if self.released {
return;
}
let mut inner = lock(&self.stream.handle.inner);
inner.process_live -= self.bytes;
inner.worker_live -= self.bytes;
inner.process_role_live[self.role as usize] -= self.bytes;
inner.worker_role_live[self.role as usize] -= self.bytes;
let snapshot = if let Some(stream) = inner.streams.get_mut(&self.stream.handle.id) {
stream.live -= self.bytes;
stream.role_live[self.role as usize] -= self.bytes;
snapshot_for_observation(&inner, self.stream.handle.id).ok()
} else {
None
};
drop(inner);
if let Some(snapshot) = snapshot {
self.stream.observe_memory(self.role, snapshot);
}
self.released = true;
}
}
impl Drop for Reservation {
fn drop(&mut self) {
self.release();
}
}
#[derive(Debug)]
pub struct ChargedBytes {
bytes: Bytes,
reservation: Reservation,
metadata: BodyMetadataOwner,
}
/// Reserve-before-allocate builder for transforms whose exact or maximum
/// output size is known before materialization.
pub struct ChargedBytesBuilder {
bytes: Vec<u8>,
capacity: usize,
reservation: Reservation,
}
impl ChargedBytesBuilder {
pub fn new(
budget: &StreamBudget,
role: MemoryRole,
capacity: usize,
) -> Result<Self, BodyError> {
let reservation = budget.reserve(role, capacity)?;
Ok(Self {
bytes: Vec::with_capacity(capacity),
capacity,
reservation,
})
}
pub fn extend_from_slice(&mut self, bytes: &[u8]) -> Result<(), BodyError> {
if self.bytes.len().saturating_add(bytes.len()) > self.capacity {
return Err(BodyError::BodyLimitExceeded);
}
self.bytes.extend_from_slice(bytes);
Ok(())
}
pub fn push(&mut self, byte: u8) -> Result<(), BodyError> {
if self.bytes.len() == self.capacity {
return Err(BodyError::BodyLimitExceeded);
}
self.bytes.push(byte);
Ok(())
}
pub fn len(&self) -> usize {
self.bytes.len()
}
pub fn is_empty(&self) -> bool {
self.bytes.is_empty()
}
pub fn capacity(&self) -> usize {
self.capacity
}
pub fn resize_zeroed(&mut self, len: usize) -> Result<(), BodyError> {
if len > self.capacity {
return Err(BodyError::BodyLimitExceeded);
}
self.bytes.resize(len, 0);
Ok(())
}
pub fn as_mut_slice(&mut self) -> &mut [u8] {
&mut self.bytes
}
pub fn truncate(&mut self, len: usize) {
self.bytes.truncate(len);
}
pub fn finish(self) -> ChargedBytes {
ChargedBytes {
bytes: Bytes::from(self.bytes),
reservation: self.reservation,
metadata: BodyMetadataOwner::default(),
}
}
}
struct DropTrackedBytesOwner {
bytes: Bytes,
_reservation: Reservation,
_metadata: BodyMetadataOwner,
}
impl AsRef<[u8]> for DropTrackedBytesOwner {
fn as_ref(&self) -> &[u8] {
&self.bytes
}
}
/// Fixed-capacity, byte-bounded owner for charged body chunks. Queue metadata
/// is reserved up front so an "empty" mailbox cannot hide unaccounted heap
/// capacity, and every stored chunk must already carry the queue's role.
#[derive(Debug)]
pub struct ChargedBodyQueue {
chunks: VecDeque<ChargedBytes>,
role: MemoryRole,
max_chunks: usize,
max_chunk_bytes: usize,
max_visible_bytes: usize,
visible_bytes: usize,
high_water_bytes: usize,
high_water_chunks: usize,
metadata_reservation: Option<Reservation>,
}
impl ChargedBodyQueue {
pub fn new(
budget: &StreamBudget,
role: MemoryRole,
plan: &BodyPlan,
hard_total_limit: usize,
max_chunks: usize,
) -> Result<Self, BodyError> {
plan.validate()?;
if hard_total_limit == 0 || max_chunks == 0 {
return Err(BodyError::InvalidPlanLimit);
}
let max_chunk_bytes = plan.max_chunk_bytes().min(hard_total_limit);
let max_visible_bytes = plan
.max_retained_bytes()
.unwrap_or(hard_total_limit)
.min(hard_total_limit);
// Account for the requested fixed queue before asking the allocator
// for its backing. A rejected stream therefore cannot transiently
// allocate an uncharged mailbox.
let metadata_bytes = max_chunks
.checked_mul(size_of::<ChargedBytes>())
.ok_or(BodyError::BudgetExceeded)?;
let metadata_reservation = budget.reserve(role, metadata_bytes)?;
// Allocate the fixed backing only after its complete logical capacity
// has been admitted. This keeps allocator capacity and accounting in
// lockstep even for queues containing only zero-byte units.
let chunks = VecDeque::with_capacity(max_chunks);
Ok(Self {
chunks,
role,
max_chunks,
max_chunk_bytes,
max_visible_bytes,
visible_bytes: 0,
high_water_bytes: 0,
high_water_chunks: 0,
metadata_reservation: Some(metadata_reservation),
})
}
pub fn push_back(&mut self, chunk: ChargedBytes) -> Result<(), BodyError> {
if chunk.role() != self.role {
return Err(BodyError::WrongMemoryRole);
}
if chunk.bytes().len() > self.max_chunk_bytes || self.chunks.len() >= self.max_chunks {
return Err(BodyError::BodyLimitExceeded);
}
let next = self
.visible_bytes
.checked_add(chunk.bytes().len())
.ok_or(BodyError::BodyLimitExceeded)?;
if next > self.max_visible_bytes {
return Err(BodyError::BodyLimitExceeded);
}
self.visible_bytes = next;
self.chunks.push_back(chunk);
self.high_water_bytes = self.high_water_bytes.max(self.visible_bytes);
self.high_water_chunks = self.high_water_chunks.max(self.chunks.len());
Ok(())
}
/// A protocol prefix has no message-count quota. Grow its queue against
/// the request budget while ordinary bounded handoff queues keep theirs.
pub fn push_back_growing(
&mut self,
budget: &StreamBudget,
chunk: ChargedBytes,
) -> Result<(), BodyError> {
if self.chunks.len() == self.max_chunks {
let capacity = self
.max_chunks
.checked_mul(2)
.ok_or(BodyError::BodyLimitExceeded)?;
let bytes = capacity
.checked_mul(size_of::<ChargedBytes>())
.ok_or(BodyError::BodyLimitExceeded)?;
let reservation = budget.reserve(self.role, bytes)?;
let mut replacement = VecDeque::with_capacity(capacity);
replacement.extend(self.chunks.drain(..));
self.chunks = replacement;
self.metadata_reservation = Some(reservation);
self.max_chunks = capacity;
}
self.push_back(chunk)
}
pub fn pop_front(&mut self) -> Option<ChargedBytes> {
let chunk = self.chunks.pop_front()?;
self.visible_bytes -= chunk.bytes().len();
Some(chunk)
}
pub fn chunks(&self) -> impl Iterator<Item = &ChargedBytes> {
self.chunks.iter()
}
pub fn len(&self) -> usize {
self.chunks.len()
}
pub fn is_empty(&self) -> bool {
self.chunks.is_empty()
}
pub fn visible_bytes(&self) -> usize {
self.visible_bytes
}
pub fn high_water_bytes(&self) -> usize {
self.high_water_bytes
}
pub fn high_water_chunks(&self) -> usize {
self.high_water_chunks
}
pub fn max_chunk_bytes(&self) -> usize {
self.max_chunk_bytes
}
pub fn allocated_chunk_capacity(&self) -> usize {
self.chunks.capacity()
}
pub fn clear_and_release(&mut self) {
// Free both elements and allocation before releasing its accounting;
// VecDeque::clear would retain capacity while reporting zero live
// metadata bytes.
let chunks = std::mem::take(&mut self.chunks);
self.visible_bytes = 0;
drop(chunks);
self.metadata_reservation.take();
}
}
impl ChargedBytes {
/// Copies an opaque transport slice into an exact-sized allocation. This
/// avoids charging a tiny visible slice while pinning unknown large backing.
pub fn copy_from_opaque(
budget: &StreamBudget,
role: MemoryRole,
bytes: &[u8],
) -> Result<Self, BodyError> {
let reservation = budget.reserve(role, bytes.len())?;
let mut exact = Vec::with_capacity(bytes.len());
exact.extend_from_slice(bytes);
Ok(Self {
bytes: Bytes::from(exact),
reservation,
metadata: BodyMetadataOwner::default(),
})
}
pub fn from_exact_vec(
budget: &StreamBudget,
role: MemoryRole,
mut bytes: Vec<u8>,
) -> Result<Self, BodyError> {
if bytes.capacity() != bytes.len() {
bytes.shrink_to_fit();
}
let retained_capacity = bytes.capacity();
let reservation = budget.reserve(role, retained_capacity)?;
Ok(Self {
bytes: Bytes::from(bytes),
reservation,
metadata: BodyMetadataOwner::default(),
})
}
pub fn bytes(&self) -> &Bytes {
&self.bytes
}
pub fn retained_capacity(&self) -> usize {
self.reservation.bytes()
}
pub fn role(&self) -> MemoryRole {
self.reservation.role()
}
pub(crate) fn metadata(&self) -> BodyMetadataOwner {
self.metadata.clone()
}
pub fn transfer_role(mut self, role: MemoryRole) -> Result<Self, BodyError> {
self.reservation.transfer_role(role)?;
Ok(self)
}
pub(crate) fn with_metadata(mut self, metadata: BodyMetadataOwner) -> Self {
self.metadata = metadata;
self
}
/// Moves both bytes and their reservation into a `Bytes` owner. Any codec
/// clone keeps the reservation alive, and the charge is released exactly
/// when the final transport clone drops.
pub fn into_tracked_bytes(self) -> Bytes {
Bytes::from_owner(DropTrackedBytesOwner {
bytes: self.bytes,
_reservation: self.reservation,
_metadata: self.metadata,
})
}
pub(crate) fn into_drop_tracked_bytes(self) -> Bytes {
self.into_tracked_bytes()
}
}
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub struct BudgetTreeSnapshot {
pub process_live: usize,
pub process_peak: usize,
pub worker_live: usize,
pub worker_peak: usize,
pub active_streams: usize,
}
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub struct StreamBudgetSnapshot {
pub live: usize,
pub peak: usize,
pub rejected: usize,
pub role_live: [usize; 9],
pub role_peak: [usize; 9],
}
/// One atomic view of the stream and both enclosing admission levels. It is
/// captured under the budget-tree mutex so telemetry never reconstructs a
/// process/worker gauge by adding racing per-request snapshots.
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub struct BudgetMemorySnapshot {
pub stream: StreamBudgetSnapshot,
pub process_rejected: usize,
pub worker_rejected: usize,
pub process_role_live: [usize; 9],
pub process_role_peak: [usize; 9],
pub worker_role_live: [usize; 9],
pub worker_role_peak: [usize; 9],
}
fn snapshot_from_stream(stream: &StreamState) -> StreamBudgetSnapshot {
StreamBudgetSnapshot {
live: stream.live,
peak: stream.peak,
rejected: stream.rejected,
role_live: stream.role_live,
role_peak: stream.role_peak,
}
}
fn snapshot_for_observation(
state: &BudgetState,
stream_id: StreamBudgetId,
) -> Result<BudgetMemorySnapshot, BodyError> {
let stream = state
.streams
.get(&stream_id)
.ok_or(BodyError::UnknownStreamBudget)?;
Ok(BudgetMemorySnapshot {
stream: snapshot_from_stream(stream),
process_rejected: state.process_rejected,
worker_rejected: state.worker_rejected,
process_role_live: state.process_role_live,
process_role_peak: state.process_role_peak,
worker_role_live: state.worker_role_live,
worker_role_peak: state.worker_role_peak,
})
}
pub(super) fn lock<T>(mutex: &Mutex<T>) -> std::sync::MutexGuard<'_, T> {
mutex
.lock()
.unwrap_or_else(|poisoned| poisoned.into_inner())
}