Skip to content

Solution for long term memory for agent coding CLIs and to facilitate handoff between different agent vendors

Rust

test(store): drive the CJK backfill cursor across batches

Vinícius DS Andrade0ea7ce0d6a74
diff --git a/crates/ai-memory-store/Cargo.toml b/crates/ai-memory-store/Cargo.toml
index ea92b839..d7fc87cc 100644
--- a/crates/ai-memory-store/Cargo.toml
+++ b/crates/ai-memory-store/Cargo.toml
@@ -40,6 +40,10 @@ fs2.workspace = true
[dev-dependencies]
tempfile.workspace = true
tracing-subscriber.workspace = true
+# The CJK backfill test counts SQLite virtual-machine steps through the
+# progress handler, which rusqlite gates behind `hooks`; the feature adds no
+# C flags and only test builds enable it.
+rusqlite = { workspace = true, features = ["hooks"] }
[lints]
workspace = true
diff --git a/crates/ai-memory-store/src/ops.rs b/crates/ai-memory-store/src/ops.rs
index 629bdfc1..9709d96f 100644
--- a/crates/ai-memory-store/src/ops.rs
+++ b/crates/ai-memory-store/src/ops.rs
@@ -11650,6 +11650,122 @@ pub(crate) mod tests {
);
}
+ /// Two bare source tables under the V87 derived indexes and triggers,
+ /// holding `rows` pages and as many observations with no derived row yet:
+ /// the shape of a store opened for the first time after that upgrade.
+ fn unindexed_cjk_fixture(rows: i64) -> Connection {
+ let conn = Connection::open_in_memory().unwrap();
+ conn.execute_batch(
+ "CREATE TABLE pages (title TEXT, body TEXT, path_search TEXT);
+ CREATE TABLE observations (title TEXT, body TEXT);",
+ )
+ .unwrap();
+ conn.execute_batch(include_str!("../migrations/V87__cjk_search_indexes.sql")).unwrap();
+ for i in 0..rows {
+ conn.execute(
+ "INSERT INTO pages (title, body, path_search) VALUES (?1, ?2, 'notes')",
+ params![format!("page {i}"), format!("支付服务 {i}")],
+ )
+ .unwrap();
+ conn.execute(
+ "INSERT INTO observations (title, body) VALUES ('prompt', ?1)",
+ [format!("需要恢复索引 {i}")],
+ )
+ .unwrap();
+ }
+ conn
+ }
+
+ /// Source rows still without a derived row, as `(pages, observations)`.
+ fn missing_cjk_rows(conn: &Connection) -> (i64, i64) {
+ let count = |sql: &str| -> i64 { conn.query_row(sql, [], |r| r.get(0)).unwrap() };
+ (
+ count(
+ "SELECT COUNT(*) FROM pages p \
+ WHERE NOT EXISTS (SELECT 1 FROM pages_cjk_fts f WHERE f.rowid = p.rowid)",
+ ),
+ count(
+ "SELECT COUNT(*) FROM observations o \
+ WHERE NOT EXISTS (SELECT 1 FROM observations_cjk_fts f WHERE f.rowid = o.rowid)",
+ ),
+ )
+ }
+
+ /// The cursor must pick up a missing row that follows an indexed one
+ /// inside a batch window, stop after the last one, and leave a repaired
+ /// store alone on the next run.
+ #[test]
+ fn cjk_backfill_resumes_past_indexed_rows_in_small_batches() {
+ let mut conn = unindexed_cjk_fixture(7);
+ // What an interrupted earlier run leaves behind: a prefix, a row in
+ // the middle and the last row already carry their derived rows.
+ for rowid in [1, 2, 4, 7] {
+ conn.execute(
+ "INSERT INTO pages_cjk_fts (rowid, title, body, path_search) \
+ VALUES (?1, '', '', '')",
+ [rowid],
+ )
+ .unwrap();
+ }
+ for rowid in [3, 6] {
+ conn.execute(
+ "INSERT INTO observations_cjk_fts (rowid, title, body) VALUES (?1, '', '')",
+ [rowid],
+ )
+ .unwrap();
+ }
+ assert_eq!(missing_cjk_rows(&conn), (3, 5));
+
+ let summary = backfill_cjk_indexes_in_batches(&mut conn, 2).unwrap();
+ assert_eq!((summary.pages, summary.observations), (3, 5));
+ assert_eq!(missing_cjk_rows(&conn), (0, 0));
+ // The repaired rows carry their terms, not placeholders.
+ let searchable: i64 = conn
+ .query_row(
+ "SELECT COUNT(*) FROM pages_cjk_fts WHERE pages_cjk_fts MATCH 'body:支付'",
+ [],
+ |r| r.get(0),
+ )
+ .unwrap();
+ assert_eq!(searchable, 3);
+
+ let again = backfill_cjk_indexes(&mut conn).unwrap();
+ assert_eq!(again, CjkBackfillSummary::default());
+ }
+
+ /// Doubling the rows must not quadruple the work. Each batch resumes
+ /// after the last rowid it indexed instead of probing the indexed prefix
+ /// again, which made the pass quadratic on a large upgraded store (#1652).
+ /// SQLite's progress handler counts virtual-machine steps: a measure of
+ /// the rows visited that, unlike a clock, reads the same on every machine.
+ #[test]
+ fn cjk_backfill_work_grows_linearly_with_the_row_count() {
+ use std::sync::atomic::{AtomicUsize, Ordering};
+
+ let steps = |rows: i64| -> usize {
+ let mut conn = unindexed_cjk_fixture(rows);
+ let visits = Arc::new(AtomicUsize::new(0));
+ let counter = Arc::clone(&visits);
+ conn.progress_handler(
+ 1,
+ Some(move || {
+ counter.fetch_add(1, Ordering::Relaxed);
+ false
+ }),
+ );
+ let summary = backfill_cjk_indexes_in_batches(&mut conn, 1).unwrap();
+ assert_eq!((summary.pages, summary.observations), (rows as u64, rows as u64));
+ visits.load(Ordering::Relaxed)
+ };
+
+ let small = steps(100);
+ let large = steps(200);
+ assert!(
+ large < small * 3,
+ "doubling the rows must not triple the work: {small} steps for 100, {large} for 200"
+ );
+ }
+
/// A checkpoint a reader holds back must not fail the operation it follows:
/// the `VACUUM` has already committed. It is logged with the
/// `(busy, log, checkpointed)` triple, left for a later checkpoint, and