Solution for long term memory for agent coding CLIs and to facilitate handoff between different agent vendors
Rust
fix(store): include the smallest rowid in the first CJK backfill batch
Djalma Júnior3b9ff3ccce22
diff --git a/crates/ai-memory-store/src/ops.rs b/crates/ai-memory-store/src/ops.rsindex abf7c298..c6fd320b 100644--- a/crates/ai-memory-store/src/ops.rs+++ b/crates/ai-memory-store/src/ops.rs@@ -1684,11 +1684,13 @@ pub(crate) fn backfill_cjk_indexes(conn: &mut Connection) -> StoreResult<CjkBack////// Each batch resumes after the highest rowid the previous one indexed. A/// batch returns the missing rows in rowid order and indexes every one of-/// them before it commits, so no row at or below the cursor is still missing-/// and the anti-join probes each source row once. Without the cursor every-/// batch filtered from the first rowid again and re-probed the whole indexed-/// prefix, so the pass slowed down as it went: quadratic in the row count,-/// hours of startup on a store with 1.6 million observations (#1652).+/// them before it commits, so no row below the cursor is still missing and+/// the anti-join probes each source row once. The cursor is an inclusive+/// lower bound, so the first batch also covers a row at `i64::MIN`. Without+/// the cursor every batch filtered from the first rowid again and re-probed+/// the whole indexed prefix, so the pass slowed down as it went: quadratic in+/// the row count, hours of startup on a store with 1.6 million observations+/// (#1652).pub(crate) fn backfill_cjk_indexes_in_batches(conn: &mut Connection,batch: i64,@@ -1698,17 +1700,17 @@ pub(crate) fn backfill_cjk_indexes_in_batches(let batch = batch.max(1);let mut summary = CjkBackfillSummary::default();- let mut after = i64::MIN;+ let mut from = i64::MIN;loop {let tx = conn.transaction()?;let rows: Vec<PageRow> = {let mut stmt = tx.prepare("SELECT p.rowid, p.title, p.body, p.path_search FROM pages p \- WHERE p.rowid > ?1 \+ WHERE p.rowid >= ?1 \AND NOT EXISTS (SELECT 1 FROM pages_cjk_fts f WHERE f.rowid = p.rowid) \ORDER BY p.rowid LIMIT ?2",)?;- stmt.query_map(params![after, batch], |row| {+ stmt.query_map(params![from, batch], |row| {Ok((row.get(0)?, row.get(1)?, row.get(2)?, row.get(3)?))})?.collect::<Result<_, _>>()?@@ -1719,24 +1721,24 @@ pub(crate) fn backfill_cjk_indexes_in_batches(}for (rowid, title, body, path_search) in &rows {index_page_cjk(&tx, *rowid, title, body, path_search)?;- after = *rowid;+ from = rowid.saturating_add(1);}summary.pages += rows.len() as u64;tx.commit()?;conn.execute_batch("PRAGMA wal_checkpoint(TRUNCATE);").ok();}- after = i64::MIN;+ from = i64::MIN;loop {let tx = conn.transaction()?;let rows: Vec<ObservationRow> = {let mut stmt = tx.prepare("SELECT o.rowid, o.title, o.body FROM observations o \- WHERE o.rowid > ?1 AND NOT EXISTS ( \+ WHERE o.rowid >= ?1 AND NOT EXISTS ( \SELECT 1 FROM observations_cjk_fts f WHERE f.rowid = o.rowid \) ORDER BY o.rowid LIMIT ?2",)?;- stmt.query_map(params![after, batch], |row| {+ stmt.query_map(params![from, batch], |row| {Ok((row.get(0)?, row.get(1)?, row.get(2)?))})?.collect::<Result<_, _>>()?@@ -1747,7 +1749,7 @@ pub(crate) fn backfill_cjk_indexes_in_batches(}for (rowid, title, body) in &rows {index_observation_cjk(&tx, *rowid, title, body)?;- after = *rowid;+ from = rowid.saturating_add(1);}summary.observations += rows.len() as u64;tx.commit()?;@@ -11868,6 +11870,36 @@ pub(crate) mod tests {assert_eq!(again, CjkBackfillSummary::default());}+ /// The cursor starts at the smallest rowid SQLite can store, so a row+ /// there (and one at the other end) is repaired like any other.+ #[test]+ fn cjk_backfill_covers_the_extreme_rowids() {+ let mut conn = unindexed_cjk_fixture(0);+ for rowid in [i64::MIN, i64::MAX] {+ conn.execute(+ "INSERT INTO pages (rowid, title, body, path_search) \+ VALUES (?1, 'edge', '支付服务', 'notes')",+ [rowid],+ )+ .unwrap();+ conn.execute(+ "INSERT INTO observations (rowid, title, body) \+ VALUES (?1, 'prompt', '需要恢复索引')",+ [rowid],+ )+ .unwrap();+ }+ assert_eq!(missing_cjk_rows(&conn), (2, 2));++ let summary = backfill_cjk_indexes_in_batches(&mut conn, 1).unwrap();+ assert_eq!((summary.pages, summary.observations), (2, 2));+ assert_eq!(missing_cjk_rows(&conn), (0, 0));+ assert_eq!(+ backfill_cjk_indexes(&mut conn).unwrap(),+ CjkBackfillSummary::default()+ );+ }+/// Doubling the rows must not quadruple the work. Each batch resumes/// after the last rowid it indexed instead of probing the indexed prefix/// again, which made the pass quadratic on a large upgraded store (#1652).