Skip to content

Make Codex and Claude Code subscriptions go further: a local model router that gives routine agent work to economical models and keeps your strongest model for the hard parts.

12.3 KB
Raw Blame
//! One bounded maintenance loop per production store. Request producers never
//! wait for this thread; its only inputs are already committed observations.
use crate::LocalObservationStore;
use std::sync::{Arc, Weak, mpsc};
use std::time::{Duration, Instant};

const MAINTENANCE_INTERVAL: Duration = Duration::from_millis(250);
const BLOB_GC_IDLE_INTERVAL: Duration = Duration::from_secs(5);
const BLOB_GC_BATCH: usize = 32;

pub struct ObservationMaintenance {
    stop: mpsc::Sender<()>,
    worker: Option<std::thread::JoinHandle<()>>,
}

/// Optional product-owned work attached to the one maintenance lifecycle.  The hook receives no
/// observation lock or deletion authority; implementations must perform their own bounded work.
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub struct ObservationMaintenanceHookError;

pub trait ObservationMaintenanceHook: Send + Sync {
    fn cycle(&self, now_ms: i64) -> Result<(), ObservationMaintenanceHookError>;
}

impl ObservationMaintenance {
    pub fn start(store: &Arc<LocalObservationStore>) -> std::io::Result<Self> {
        Self::start_with_hook(store, None)
    }

    pub fn start_with_hook(
        store: &Arc<LocalObservationStore>,
        hook: Option<Arc<dyn ObservationMaintenanceHook>>,
    ) -> std::io::Result<Self> {
        let lease = LocalWorkerGuard::acquire(&store.maintenance_running).ok_or_else(|| {
            std::io::Error::new(
                std::io::ErrorKind::AlreadyExists,
                "observation maintenance already running",
            )
        })?;
        let (stop, receiver) = mpsc::channel();
        let weak = Arc::downgrade(store);
        let result = std::thread::Builder::new()
            .name("hiroute-observation-maintenance".into())
            .spawn(move || {
                let _lease = lease;
                run(weak, receiver, hook)
            });
        let worker = result.inspect_err(|_| {
            store
                .maintenance_errors
                .fetch_add(1, std::sync::atomic::Ordering::Relaxed);
        })?;
        Ok(Self {
            stop,
            worker: Some(worker),
        })
    }
}

impl Drop for ObservationMaintenance {
    fn drop(&mut self) {
        let _ = self.stop.send(());
        // Finish the current bounded cycle before releasing the owner. A hook may
        // hold the product stores, and an immediate reopen must not race that lease.
        if let Some(worker) = self.worker.take() {
            let _ = worker.join();
        }
    }
}

fn run(
    store: Weak<LocalObservationStore>,
    stop: mpsc::Receiver<()>,
    hook: Option<Arc<dyn ObservationMaintenanceHook>>,
) {
    let mut index = store
        .upgrade()
        .and_then(|store| crate::text_index::TextIndexBuilder::new(&store).ok());
    let mut blob_gc = BlobGcSchedule {
        next_due: Instant::now(),
    };
    let mut interval = MAINTENANCE_INTERVAL;
    while matches!(
        stop.recv_timeout(interval),
        Err(mpsc::RecvTimeoutError::Timeout)
    ) {
        let Some(store) = store.upgrade() else {
            break;
        };
        let capacity_failed = store
            .capacity_cycle(
                std::time::SystemTime::now()
                    .duration_since(std::time::UNIX_EPOCH)
                    .ok()
                    .and_then(|t| i64::try_from(t.as_millis()).ok())
                    .unwrap_or(0),
            )
            .is_err();
        let backfill_failed = store.backfill_inline_payloads().is_err();
        let settlement_failed = store.settle_pending_valuations(16).is_err();
        let now_ms = std::time::SystemTime::now()
            .duration_since(std::time::UNIX_EPOCH)
            .ok()
            .and_then(|time| i64::try_from(time.as_millis()).ok());
        // Reuse this lifecycle without limiting catch-up to the settlement
        // tick's sixteen requests. Release the writer after each <=200 batch.
        let analysis_start = Instant::now();
        let mut analysis_failed = false;
        let mut analysis_backlog = false;
        if let Some(now) = now_ms {
            while analysis_start.elapsed() < Duration::from_millis(50) {
                match store.reconcile_analysis_batch(now) {
                    Ok(0) => {
                        analysis_backlog = false;
                        break;
                    }
                    Ok(_) => {
                        analysis_backlog = true;
                        std::thread::yield_now()
                    }
                    Err(_) => {
                        analysis_failed = true;
                        break;
                    }
                }
            }
        }
        interval = if analysis_backlog && !analysis_failed {
            Duration::from_millis(1)
        } else {
            MAINTENANCE_INTERVAL
        };
        let retention_failed = now_ms
            .map(|now| {
                store.managed_text_expire_pending(now, 16).is_err()
                    | store.managed_text_gc(now, 32).is_err()
                    | store.managed_text_progress_gc(now, 32).is_err()
                    | store.expire_request_details(now, 16).is_err()
            })
            .unwrap_or(true);
        // Only physical blob collection backs off. Expiry, indexing, settlement and hooks
        // retain their existing cadence; explicit user deletion still collects immediately.
        let gc_failed = blob_gc
            .run_if_due(Instant::now, || store.collect_garbage_batch(BLOB_GC_BATCH))
            .is_err();
        let hook_failed = match (hook.as_ref(), now_ms) {
            (Some(hook), Some(now)) => hook.cycle(now).is_err(),
            (Some(_), None) => true,
            (None, _) => false,
        };
        let index_failed = match index.as_mut() {
            Some(index) => index.cycle(&store).is_err(),
            None => {
                index = crate::text_index::TextIndexBuilder::new(&store).ok();
                index.is_none()
            }
        };
        if capacity_failed
            || backfill_failed
            || settlement_failed
            || retention_failed
            || gc_failed
            || hook_failed
            || index_failed
            || analysis_failed
        {
            store
                .maintenance_errors
                .fetch_add(1, std::sync::atomic::Ordering::Relaxed);
        }
    }
}

/// A full batch may leave a backlog. Idle and failed passes wait longer, measured
/// from completion so a slow query cannot immediately schedule another expensive pass.
struct BlobGcSchedule {
    next_due: Instant,
}

impl BlobGcSchedule {
    fn run_if_due(
        &mut self,
        clock: impl Fn() -> Instant,
        collect: impl FnOnce() -> Result<u64, hiroute_domain::ObservationQueryError>,
    ) -> Result<(), hiroute_domain::ObservationQueryError> {
        if clock() < self.next_due {
            return Ok(());
        }
        let result = collect();
        let interval = if matches!(result, Ok(count) if count == BLOB_GC_BATCH as u64) {
            MAINTENANCE_INTERVAL
        } else {
            BLOB_GC_IDLE_INTERVAL
        };
        self.next_due = clock() + interval;
        result.map(|_| ())
    }
}

/// Process-local ownership only: not a distributed lease or durable authority.
pub(crate) struct LocalWorkerGuard(std::sync::Arc<std::sync::atomic::AtomicBool>);
impl LocalWorkerGuard {
    pub(crate) fn acquire(flag: &std::sync::Arc<std::sync::atomic::AtomicBool>) -> Option<Self> {
        flag.compare_exchange(
            false,
            true,
            std::sync::atomic::Ordering::AcqRel,
            std::sync::atomic::Ordering::Acquire,
        )
        .ok()
        .map(|_| Self(flag.clone()))
    }
}
impl Drop for LocalWorkerGuard {
    fn drop(&mut self) {
        self.0.store(false, std::sync::atomic::Ordering::Release);
    }
}
impl LocalObservationStore {
    pub fn maintenance_status(&self) -> (bool, u64) {
        (
            self.maintenance_running
                .load(std::sync::atomic::Ordering::Acquire),
            self.maintenance_errors
                .load(std::sync::atomic::Ordering::Relaxed),
        )
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use hiroute_domain::ObservationQueryError;
    use std::cell::Cell;

    #[test]
    fn blob_gc_idle_partial_and_failed_passes_wait_five_seconds() {
        for outcome in [
            Ok(0),
            Ok(1),
            Ok(31),
            Err(ObservationQueryError::Unavailable),
        ] {
            let start = Instant::now();
            let mut gc = BlobGcSchedule { next_due: start };
            let failed = outcome.is_err();
            assert_eq!(gc.run_if_due(|| start, || outcome).is_err(), failed);
            // Other maintenance may tick twenty times without repeating GC.
            for tick in 1..20 {
                gc.run_if_due(
                    || start + MAINTENANCE_INTERVAL * tick,
                    || panic!("GC ran before its idle/retry deadline"),
                )
                .unwrap();
            }
            let called = Cell::new(false);
            gc.run_if_due(
                || start + BLOB_GC_IDLE_INTERVAL,
                || {
                    called.set(true);
                    Ok(0)
                },
            )
            .unwrap();
            assert!(called.get());
        }
    }

    #[test]
    fn blob_gc_full_batches_continue_and_deadlines_start_at_completion() {
        let start = Instant::now();
        let clock = Cell::new(start);
        let mut gc = BlobGcSchedule { next_due: start };
        gc.run_if_due(
            || clock.get(),
            || {
                clock.set(start + Duration::from_secs(2));
                Ok(BLOB_GC_BATCH as u64)
            },
        )
        .unwrap();
        clock.set(start + Duration::from_secs(2) + Duration::from_millis(249));
        gc.run_if_due(|| clock.get(), || panic!("early backlog retry"))
            .unwrap();
        clock.set(clock.get() + Duration::from_millis(1));
        let called = Cell::new(false);
        gc.run_if_due(
            || clock.get(),
            || {
                called.set(true);
                clock.set(clock.get() + Duration::from_secs(6));
                Err(ObservationQueryError::Unavailable)
            },
        )
        .unwrap_err();
        assert!(called.get());
        clock.set(clock.get() + BLOB_GC_IDLE_INTERVAL - Duration::from_millis(1));
        gc.run_if_due(|| clock.get(), || panic!("retry measured from start"))
            .unwrap();
        clock.set(clock.get() + Duration::from_millis(1));
        gc.run_if_due(|| clock.get(), || Ok(0)).unwrap();
    }

    struct BlockedCycle {
        entered: mpsc::Sender<()>,
        release: std::sync::Mutex<mpsc::Receiver<()>>,
    }

    impl ObservationMaintenanceHook for BlockedCycle {
        fn cycle(&self, _: i64) -> Result<(), ObservationMaintenanceHookError> {
            self.entered.send(()).unwrap();
            self.release
                .lock()
                .unwrap()
                .recv_timeout(Duration::from_secs(5))
                .unwrap();
            Ok(())
        }
    }

    #[test]
    fn dropping_maintenance_finishes_its_current_cycle_and_releases_the_lease() {
        let root = tempfile::tempdir().unwrap();
        let store = Arc::new(
            LocalObservationStore::open(root.path(), crate::DigestAuthority::new([1; 32])).unwrap(),
        );
        let (entered, did_enter) = mpsc::channel();
        let (release, wait_for_release) = mpsc::channel();
        let worker = ObservationMaintenance::start_with_hook(
            &store,
            Some(Arc::new(BlockedCycle {
                entered,
                release: std::sync::Mutex::new(wait_for_release),
            })),
        )
        .unwrap();
        did_enter.recv_timeout(Duration::from_secs(5)).unwrap();
        let (dropped, did_drop) = mpsc::channel();
        let owner = std::thread::spawn(move || {
            drop(worker);
            dropped.send(()).unwrap();
        });
        assert!(matches!(
            did_drop.recv_timeout(Duration::from_millis(50)),
            Err(mpsc::RecvTimeoutError::Timeout)
        ));
        release.send(()).unwrap();
        did_drop.recv_timeout(Duration::from_secs(5)).unwrap();
        owner.join().unwrap();
        assert!(!store.maintenance_status().0);
        drop(ObservationMaintenance::start(&store).unwrap());
        assert!(!store.maintenance_status().0);
    }
}