Skip to content

Make Codex and Claude Code subscriptions go further: a local model router that gives routine agent work to economical models and keeps your strongest model for the hard parts.

fix: reduce idle observation blob GC overhead (#35)

澄潭da1506dbdd87
diff --git a/crates/observation/src/maintenance.rs b/crates/observation/src/maintenance.rs
index d9ca7945..9d574f2e 100644
--- a/crates/observation/src/maintenance.rs
+++ b/crates/observation/src/maintenance.rs
@@ -2,7 +2,11 @@
//! wait for this thread; its only inputs are already committed observations.
use crate::LocalObservationStore;
use std::sync::{Arc, Weak, mpsc};
-use std::time::Duration;
+use std::time::{Duration, Instant};
+
+const MAINTENANCE_INTERVAL: Duration = Duration::from_millis(250);
+const BLOB_GC_IDLE_INTERVAL: Duration = Duration::from_secs(5);
+const BLOB_GC_BATCH: usize = 32;
pub struct ObservationMaintenance {
stop: mpsc::Sender<()>,
@@ -64,8 +68,11 @@ fn run(
let mut index = store
.upgrade()
.and_then(|store| crate::text_index::TextIndexBuilder::new(&store).ok());
+ let mut blob_gc = BlobGcSchedule {
+ next_due: Instant::now(),
+ };
while matches!(
- stop.recv_timeout(Duration::from_millis(250)),
+ stop.recv_timeout(MAINTENANCE_INTERVAL),
Err(mpsc::RecvTimeoutError::Timeout)
) {
let Some(store) = store.upgrade() else {
@@ -83,9 +90,13 @@ fn run(
| store.managed_text_gc(now, 32).is_err()
| store.managed_text_progress_gc(now, 32).is_err()
| store.expire_request_details(now, 16).is_err()
- | store.collect_garbage_batch(32).is_err()
})
.unwrap_or(true);
+ // Only physical blob collection backs off. Expiry, indexing, settlement and hooks
+ // retain their existing cadence; explicit user deletion still collects immediately.
+ let gc_failed = blob_gc
+ .run_if_due(Instant::now, || store.collect_garbage_batch(BLOB_GC_BATCH))
+ .is_err();
let hook_failed = match (hook.as_ref(), now_ms) {
(Some(hook), Some(now)) => hook.cycle(now).is_err(),
(Some(_), None) => true,
@@ -98,7 +109,13 @@ fn run(
index.is_none()
}
};
- if backfill_failed || settlement_failed || retention_failed || hook_failed || index_failed {
+ if backfill_failed
+ || settlement_failed
+ || retention_failed
+ || gc_failed
+ || hook_failed
+ || index_failed
+ {
store
.maintenance_errors
.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
@@ -106,6 +123,32 @@ fn run(
}
}
+/// A full batch may leave a backlog. Idle and failed passes wait longer, measured
+/// from completion so a slow query cannot immediately schedule another expensive pass.
+struct BlobGcSchedule {
+ next_due: Instant,
+}
+
+impl BlobGcSchedule {
+ fn run_if_due(
+ &mut self,
+ clock: impl Fn() -> Instant,
+ collect: impl FnOnce() -> Result<u64, hiroute_domain::ObservationQueryError>,
+ ) -> Result<(), hiroute_domain::ObservationQueryError> {
+ if clock() < self.next_due {
+ return Ok(());
+ }
+ let result = collect();
+ let interval = if matches!(result, Ok(count) if count == BLOB_GC_BATCH as u64) {
+ MAINTENANCE_INTERVAL
+ } else {
+ BLOB_GC_IDLE_INTERVAL
+ };
+ self.next_due = clock() + interval;
+ result.map(|_| ())
+ }
+}
+
/// Process-local ownership only: not a distributed lease or durable authority.
pub(crate) struct LocalWorkerGuard(std::sync::Arc<std::sync::atomic::AtomicBool>);
impl LocalWorkerGuard {
@@ -135,3 +178,78 @@ impl LocalObservationStore {
)
}
}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use hiroute_domain::ObservationQueryError;
+ use std::cell::Cell;
+
+ #[test]
+ fn blob_gc_idle_partial_and_failed_passes_wait_five_seconds() {
+ for outcome in [
+ Ok(0),
+ Ok(1),
+ Ok(31),
+ Err(ObservationQueryError::Unavailable),
+ ] {
+ let start = Instant::now();
+ let mut gc = BlobGcSchedule { next_due: start };
+ let failed = outcome.is_err();
+ assert_eq!(gc.run_if_due(|| start, || outcome).is_err(), failed);
+ // Other maintenance may tick twenty times without repeating GC.
+ for tick in 1..20 {
+ gc.run_if_due(
+ || start + MAINTENANCE_INTERVAL * tick,
+ || panic!("GC ran before its idle/retry deadline"),
+ )
+ .unwrap();
+ }
+ let called = Cell::new(false);
+ gc.run_if_due(
+ || start + BLOB_GC_IDLE_INTERVAL,
+ || {
+ called.set(true);
+ Ok(0)
+ },
+ )
+ .unwrap();
+ assert!(called.get());
+ }
+ }
+
+ #[test]
+ fn blob_gc_full_batches_continue_and_deadlines_start_at_completion() {
+ let start = Instant::now();
+ let clock = Cell::new(start);
+ let mut gc = BlobGcSchedule { next_due: start };
+ gc.run_if_due(
+ || clock.get(),
+ || {
+ clock.set(start + Duration::from_secs(2));
+ Ok(BLOB_GC_BATCH as u64)
+ },
+ )
+ .unwrap();
+ clock.set(start + Duration::from_secs(2) + Duration::from_millis(249));
+ gc.run_if_due(|| clock.get(), || panic!("early backlog retry"))
+ .unwrap();
+ clock.set(clock.get() + Duration::from_millis(1));
+ let called = Cell::new(false);
+ gc.run_if_due(
+ || clock.get(),
+ || {
+ called.set(true);
+ clock.set(clock.get() + Duration::from_secs(6));
+ Err(ObservationQueryError::Unavailable)
+ },
+ )
+ .unwrap_err();
+ assert!(called.get());
+ clock.set(clock.get() + BLOB_GC_IDLE_INTERVAL - Duration::from_millis(1));
+ gc.run_if_due(|| clock.get(), || panic!("retry measured from start"))
+ .unwrap();
+ clock.set(clock.get() + Duration::from_millis(1));
+ gc.run_if_due(|| clock.get(), || Ok(0)).unwrap();
+ }
+}
diff --git a/crates/observation/src/store/retention.rs b/crates/observation/src/store/retention.rs
index bc344f1e..6d2203cf 100644
--- a/crates/observation/src/store/retention.rs
+++ b/crates/observation/src/store/retention.rs
@@ -14,6 +14,14 @@ use super::value_rollup::{
};
use super::{LocalObservationStore, increment_store_revision, store_revision};
+pub(super) const GARBAGE_CANDIDATES_SQL: &str =
+ "SELECT b.workspace_id, b.blob_digest, b.object_path FROM content_blobs_v2 b
+ WHERE NOT EXISTS (
+ SELECT 1 FROM content_instances_v2 c
+ WHERE c.workspace_id=b.workspace_id
+ AND c.content_blob_digest=b.blob_digest AND c.state='complete'
+ ) ORDER BY b.rowid LIMIT ?1";
+
impl LocalObservationStore {
pub(super) fn preview_session_deletion_base(
&self,
@@ -307,14 +315,7 @@ impl LocalObservationStore {
let mut connection = self.connection.lock();
let unreferenced = {
let mut statement = connection
- .prepare(
- "SELECT b.workspace_id, b.blob_digest, b.object_path FROM content_blobs_v2 b
- WHERE NOT EXISTS (
- SELECT 1 FROM content_instances_v2 c
- WHERE c.workspace_id=b.workspace_id
- AND c.content_blob_digest=b.blob_digest AND c.state='complete'
- ) ORDER BY b.rowid LIMIT ?1",
- )
+ .prepare(GARBAGE_CANDIDATES_SQL)
.map_err(|_| ObservationQueryError::Unavailable)?;
statement
.query_map([limit], |row| {
diff --git a/crates/observation/src/store/schema.rs b/crates/observation/src/store/schema.rs
index 7528397b..94d34f08 100644
--- a/crates/observation/src/store/schema.rs
+++ b/crates/observation/src/store/schema.rs
@@ -606,6 +606,9 @@ const CONVERSATION_CONTENT_SCHEMA_V4: &str = r#"
CREATE INDEX IF NOT EXISTS content_instances_v2_request
ON content_instances_v2(workspace_id, request_id, direction, fork_id, part_ordinal);
+ CREATE INDEX IF NOT EXISTS content_instances_v2_live_blob
+ ON content_instances_v2(workspace_id, content_blob_digest) WHERE state='complete';
+
CREATE TABLE IF NOT EXISTS content_chunks_v2 (
workspace_id TEXT NOT NULL,
content_id TEXT NOT NULL,
diff --git a/crates/observation/src/store/tests/blob_gc.rs b/crates/observation/src/store/tests/blob_gc.rs
new file mode 100644
index 00000000..86d18171
--- /dev/null
+++ b/crates/observation/src/store/tests/blob_gc.rs
@@ -0,0 +1,109 @@
+use super::support::open_store;
+use crate::LocalObservationStore;
+use rusqlite::{StatementStatus, params};
+use tempfile::tempdir;
+
+use super::super::retention::GARBAGE_CANDIDATES_SQL;
+
+fn seed(store: &LocalObservationStore, blobs: usize, references: usize) {
+ let mut connection = store.connection.lock();
+ let tx = connection.transaction().unwrap();
+ for id in 0..blobs {
+ let digest = format!("blob-{id:06}");
+ let path = store.blob_path(&digest);
+ std::fs::write(&path, b"content").unwrap();
+ tx.execute(
+ "INSERT INTO content_blobs_v2 VALUES('workspace',?1,?2,7,'text/plain','complete',1)",
+ params![digest, path.to_string_lossy()],
+ )
+ .unwrap();
+ for reference in 0..references {
+ tx.execute(
+ "INSERT INTO content_instances_v2
+ (workspace_id,content_id,message_instance_id,request_id,direction,fork_id,
+ part_ordinal,content_kind,canonical_media_type,content_blob_digest,
+ has_content_ref,accumulated_byte_count,state,created_at_unix_nanos)
+ VALUES('workspace',?1,'message',?1,'response','fork',0,'text','text/plain',?2,1,7,'complete',1)",
+ params![format!("content-{id:06}-{reference}"), digest],
+ ).unwrap();
+ }
+ }
+ tx.commit().unwrap();
+}
+
+fn idle_query_steps(store: &LocalObservationStore) -> i32 {
+ let connection = store.connection.lock();
+ let mut statement = connection.prepare(GARBAGE_CANDIDATES_SQL).unwrap();
+ assert!(!statement.exists([32]).unwrap());
+ statement.get_status(StatementStatus::VmStep)
+}
+
+#[test]
+fn blob_gc_live_lookup_stays_linear_and_existing_databases_gain_index() {
+ let root = tempdir().unwrap();
+ let store = open_store(root.path());
+ seed(&store, 1024, 2);
+ let indexed = idle_query_steps(&store);
+ // Count SQLite work rather than asserting machine-dependent wall-clock time.
+ assert!(indexed < 1024 * 50, "indexed VM steps: {indexed}");
+ store
+ .connection
+ .lock()
+ .execute_batch("DROP INDEX content_instances_v2_live_blob")
+ .unwrap();
+ let without_index = idle_query_steps(&store);
+ assert!(
+ without_index > indexed * 20,
+ "regression fixture must expose the expensive old lookup"
+ );
+ drop(store);
+
+ // Opening an existing current-schema database must install the index too;
+ // opening it again must remain idempotent and preserve all content.
+ for _ in 0..2 {
+ let reopened = open_store(root.path());
+ assert!(idle_query_steps(&reopened) < 1024 * 50);
+ assert_eq!(reopened.collect_garbage_batch(32).unwrap(), 0);
+ assert!(reopened.blob_path("blob-000000").exists());
+ }
+}
+
+#[test]
+fn blob_gc_preserves_shared_live_content_until_the_last_reference_is_removed() {
+ let root = tempdir().unwrap();
+ let store = open_store(root.path());
+ seed(&store, 1, 2);
+ let path = store.blob_path("blob-000000");
+ store
+ .connection
+ .lock()
+ .execute(
+ "DELETE FROM content_instances_v2 WHERE content_id='content-000000-0'",
+ [],
+ )
+ .unwrap();
+ assert_eq!(store.collect_garbage_batch(32).unwrap(), 0);
+ assert!(path.exists());
+ store
+ .connection
+ .lock()
+ .execute(
+ "DELETE FROM content_instances_v2 WHERE content_id='content-000000-1'",
+ [],
+ )
+ .unwrap();
+ assert_eq!(store.collect_garbage_batch(32).unwrap(), 1);
+ assert!(!path.exists());
+ assert_eq!(store.collect_garbage_batch(32).unwrap(), 0);
+}
+
+#[test]
+fn blob_gc_full_batch_leaves_a_backlog_and_partial_batch_finishes_it() {
+ let root = tempdir().unwrap();
+ let store = open_store(root.path());
+ seed(&store, 33, 0);
+ assert_eq!(store.collect_garbage_batch(32).unwrap(), 32);
+ assert_eq!(store.collect_garbage_batch(32).unwrap(), 1);
+ assert_eq!(store.collect_garbage_batch(32).unwrap(), 0);
+ assert_eq!(std::fs::read_dir(&store.blob_root).unwrap().count(), 0);
+}
diff --git a/crates/observation/src/store/tests/mod.rs b/crates/observation/src/store/tests/mod.rs
index 778e65ed..98b07f3a 100644
--- a/crates/observation/src/store/tests/mod.rs
+++ b/crates/observation/src/store/tests/mod.rs
@@ -1,3 +1,4 @@
+mod blob_gc;
mod content_v2;
mod fact_regression;
mod failure_isolation;