Solution for long term memory for agent coding CLIs and to facilitate handoff between different agent vendors
Rust
test(store): drive the CJK backfill cursor across batches
Vinícius DS Andrade0ea7ce0d6a74
diff --git a/crates/ai-memory-store/Cargo.toml b/crates/ai-memory-store/Cargo.tomlindex ea92b839..d7fc87cc 100644--- a/crates/ai-memory-store/Cargo.toml+++ b/crates/ai-memory-store/Cargo.toml@@ -40,6 +40,10 @@ fs2.workspace = true[dev-dependencies]tempfile.workspace = truetracing-subscriber.workspace = true+# The CJK backfill test counts SQLite virtual-machine steps through the+# progress handler, which rusqlite gates behind `hooks`; the feature adds no+# C flags and only test builds enable it.+rusqlite = { workspace = true, features = ["hooks"] }[lints]workspace = truediff --git a/crates/ai-memory-store/src/ops.rs b/crates/ai-memory-store/src/ops.rsindex 629bdfc1..9709d96f 100644--- a/crates/ai-memory-store/src/ops.rs+++ b/crates/ai-memory-store/src/ops.rs@@ -11650,6 +11650,122 @@ pub(crate) mod tests {);}+ /// Two bare source tables under the V87 derived indexes and triggers,+ /// holding `rows` pages and as many observations with no derived row yet:+ /// the shape of a store opened for the first time after that upgrade.+ fn unindexed_cjk_fixture(rows: i64) -> Connection {+ let conn = Connection::open_in_memory().unwrap();+ conn.execute_batch(+ "CREATE TABLE pages (title TEXT, body TEXT, path_search TEXT);+ CREATE TABLE observations (title TEXT, body TEXT);",+ )+ .unwrap();+ conn.execute_batch(include_str!("../migrations/V87__cjk_search_indexes.sql")).unwrap();+ for i in 0..rows {+ conn.execute(+ "INSERT INTO pages (title, body, path_search) VALUES (?1, ?2, 'notes')",+ params![format!("page {i}"), format!("支付服务 {i}")],+ )+ .unwrap();+ conn.execute(+ "INSERT INTO observations (title, body) VALUES ('prompt', ?1)",+ [format!("需要恢复索引 {i}")],+ )+ .unwrap();+ }+ conn+ }++ /// Source rows still without a derived row, as `(pages, observations)`.+ fn missing_cjk_rows(conn: &Connection) -> (i64, i64) {+ let count = |sql: &str| -> i64 { conn.query_row(sql, [], |r| r.get(0)).unwrap() };+ (+ count(+ "SELECT COUNT(*) FROM pages p \+ WHERE NOT EXISTS (SELECT 1 FROM pages_cjk_fts f WHERE f.rowid = p.rowid)",+ ),+ count(+ "SELECT COUNT(*) FROM observations o \+ WHERE NOT EXISTS (SELECT 1 FROM observations_cjk_fts f WHERE f.rowid = o.rowid)",+ ),+ )+ }++ /// The cursor must pick up a missing row that follows an indexed one+ /// inside a batch window, stop after the last one, and leave a repaired+ /// store alone on the next run.+ #[test]+ fn cjk_backfill_resumes_past_indexed_rows_in_small_batches() {+ let mut conn = unindexed_cjk_fixture(7);+ // What an interrupted earlier run leaves behind: a prefix, a row in+ // the middle and the last row already carry their derived rows.+ for rowid in [1, 2, 4, 7] {+ conn.execute(+ "INSERT INTO pages_cjk_fts (rowid, title, body, path_search) \+ VALUES (?1, '', '', '')",+ [rowid],+ )+ .unwrap();+ }+ for rowid in [3, 6] {+ conn.execute(+ "INSERT INTO observations_cjk_fts (rowid, title, body) VALUES (?1, '', '')",+ [rowid],+ )+ .unwrap();+ }+ assert_eq!(missing_cjk_rows(&conn), (3, 5));++ let summary = backfill_cjk_indexes_in_batches(&mut conn, 2).unwrap();+ assert_eq!((summary.pages, summary.observations), (3, 5));+ assert_eq!(missing_cjk_rows(&conn), (0, 0));+ // The repaired rows carry their terms, not placeholders.+ let searchable: i64 = conn+ .query_row(+ "SELECT COUNT(*) FROM pages_cjk_fts WHERE pages_cjk_fts MATCH 'body:支付'",+ [],+ |r| r.get(0),+ )+ .unwrap();+ assert_eq!(searchable, 3);++ let again = backfill_cjk_indexes(&mut conn).unwrap();+ assert_eq!(again, CjkBackfillSummary::default());+ }++ /// Doubling the rows must not quadruple the work. Each batch resumes+ /// after the last rowid it indexed instead of probing the indexed prefix+ /// again, which made the pass quadratic on a large upgraded store (#1652).+ /// SQLite's progress handler counts virtual-machine steps: a measure of+ /// the rows visited that, unlike a clock, reads the same on every machine.+ #[test]+ fn cjk_backfill_work_grows_linearly_with_the_row_count() {+ use std::sync::atomic::{AtomicUsize, Ordering};++ let steps = |rows: i64| -> usize {+ let mut conn = unindexed_cjk_fixture(rows);+ let visits = Arc::new(AtomicUsize::new(0));+ let counter = Arc::clone(&visits);+ conn.progress_handler(+ 1,+ Some(move || {+ counter.fetch_add(1, Ordering::Relaxed);+ false+ }),+ );+ let summary = backfill_cjk_indexes_in_batches(&mut conn, 1).unwrap();+ assert_eq!((summary.pages, summary.observations), (rows as u64, rows as u64));+ visits.load(Ordering::Relaxed)+ };++ let small = steps(100);+ let large = steps(200);+ assert!(+ large < small * 3,+ "doubling the rows must not triple the work: {small} steps for 100, {large} for 200"+ );+ }+/// A checkpoint a reader holds back must not fail the operation it follows:/// the `VACUUM` has already committed. It is logged with the/// `(busy, log, checkpointed)` triple, left for a later checkpoint, and