Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.
TypeScript agentaiai-agentsall-in-onecrmcrm-systememaillinearmcpmessagingnotesnotionofficerustslackslack-alternativestartupstartup-toolstypescriptworkspace
Seanaye/feat/batch upsert with index (#6204)
Sean Aye7b17df1381ff
diff --git a/crates/client/cache-turso/src/storage.rs b/crates/client/cache-turso/src/storage.rsindex ad66e93140..0eaff1da49 100644--- a/crates/client/cache-turso/src/storage.rs+++ b/crates/client/cache-turso/src/storage.rs@@ -91,8 +91,15 @@ const CREATE_SCHEMA: [&str; 28] = [const RECORD_GET: &str = "SELECT value FROM records WHERE __typename = ?1 AND id = ?2";const RECORD_UPSERT: &str = "INSERT INTO records (__typename, id, value) VALUES (?1, ?2, ?3) ON CONFLICT (__typename, id) DO UPDATE SET value = excluded.value";const RECORD_DELETE: &str = "DELETE FROM records WHERE __typename = ?1 AND id = ?2";-const SEARCH_DELETE: &str = "DELETE FROM search_documents WHERE __typename = ?1 AND id = ?2";+// Turso otherwise chooses the browse index's profile prefix for a direct composite-key delete.+// Force a point lookup through the existing primary-key index, then delete by its rowid.+const SEARCH_ROWID: &str = "SELECT rowid FROM search_documents INDEXED BY sqlite_autoindex_search_documents_1 WHERE profile = ? AND __typename = ? AND id = ?";+const SEARCH_DELETE: &str = "DELETE FROM search_documents WHERE rowid = ?1";const SEARCH_UPSERT: &str = "INSERT INTO search_documents (profile, __typename, id, bucket, search_text, timestamp_ms, source_hash) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7) ON CONFLICT (profile, __typename, id) DO UPDATE SET bucket = excluded.bucket, search_text = excluded.search_text, timestamp_ms = excluded.timestamp_ms, source_hash = excluded.source_hash";+const SEARCH_UPSERT_PREFIX: &str = "INSERT INTO search_documents (profile, __typename, id, bucket, search_text, timestamp_ms, source_hash) VALUES ";+const SEARCH_UPSERT_ROW: &str = "(?, ?, ?, ?, ?, ?, ?)";+const SEARCH_UPSERT_SUFFIX: &str = " ON CONFLICT (profile, __typename, id) DO UPDATE SET bucket = excluded.bucket, search_text = excluded.search_text, timestamp_ms = excluded.timestamp_ms, source_hash = excluded.source_hash";+const SEARCH_WRITE_BATCH_SIZE: usize = 100;const SEARCH_LOAD: &str = "SELECT __typename, id, bucket, search_text, timestamp_ms, source_hash FROM search_documents WHERE profile = ?1";const SEARCH_BROWSE: &str = "SELECT __typename, id, bucket, search_text, timestamp_ms, source_hash FROM search_documents INDEXED BY search_documents_browse_idx WHERE profile = ?1 AND bucket = ?2 ORDER BY timestamp_ms DESC, __typename ASC, id ASC LIMIT ?3";const SEARCH_BROWSE_AFTER: &str = "SELECT __typename, id, bucket, search_text, timestamp_ms, source_hash FROM search_documents INDEXED BY search_documents_browse_idx WHERE profile = ?1 AND bucket = ?2 AND (timestamp_ms < ?3 OR (timestamp_ms = ?3 AND (__typename > ?4 OR (__typename = ?4 AND id > ?5)))) ORDER BY timestamp_ms DESC, __typename ASC, id ASC LIMIT ?6";@@ -858,7 +865,8 @@ impl Storage for TursoStorage {let connection = self.connection();driver::write_transaction(&connection, || {let mut record_statement = driver::prepare(&connection, RECORD_DELETE)?;- let mut search_statement = driver::prepare(&connection, SEARCH_DELETE)?;+ let mut search_rowid_statement = driver::prepare(&connection, SEARCH_ROWID)?;+ let mut search_delete_statement = driver::prepare(&connection, SEARCH_DELETE)?;for (index, key) in keys.iter().enumerate() {let changed = driver::execute_prepared(&mut record_statement,@@ -867,13 +875,12 @@ impl Storage for TursoStorage {if !(0..=1).contains(&changed) {return Err(invariant());}- let search_changed = driver::execute_prepared(- &mut search_statement,- vec![text(&key.typename), text(&key.id)],+ delete_search_document(+ &mut search_rowid_statement,+ &mut search_delete_statement,+ SearchProfile::QuickAccessV1,+ key,)?;- if search_changed < 0 {- return Err(invariant());- }self.fault_after(TestFaultSite::Delete, index)?;}Ok(())@@ -1421,7 +1428,8 @@ impl PredicateIndexStorage for TursoStorage {let connection = self.connection();driver::write_transaction(&connection, || {let mut record_statement = driver::prepare(&connection, RECORD_DELETE)?;- let mut search_statement = driver::prepare(&connection, SEARCH_DELETE)?;+ let mut search_rowid_statement = driver::prepare(&connection, SEARCH_ROWID)?;+ let mut search_delete_statement = driver::prepare(&connection, SEARCH_DELETE)?;for key in &keys {let changed = driver::execute_prepared(&mut record_statement,@@ -1430,9 +1438,11 @@ impl PredicateIndexStorage for TursoStorage {if !(0..=1).contains(&changed) {return Err(invariant());}- driver::execute_prepared(- &mut search_statement,- vec![text(&key.typename), text(&key.id)],+ delete_search_document(+ &mut search_rowid_statement,+ &mut search_delete_statement,+ SearchProfile::QuickAccessV1,+ key,)?;}for key in projection_keys {@@ -2594,6 +2604,7 @@ fn initialize(RECORD_GET,RECORD_UPSERT,RECORD_DELETE,+ SEARCH_ROWID,SEARCH_DELETE,SEARCH_UPSERT,SEARCH_LOAD,@@ -4089,41 +4100,138 @@ fn prepare_records(.collect()}-fn write_search_documents(+fn delete_search_document(+ rowid_statement: &mut turso_core::Statement,+ delete_statement: &mut turso_core::Statement,+ profile: SearchProfile,+ key: &RecordKey,+) -> Result<(), TursoStorageError> {+ let rows = driver::query_prepared(+ rowid_statement,+ vec![text(profile.as_str()), text(&key.typename), text(&key.id)],+ )?;+ match rows.as_slice() {+ [] => Ok(()),+ [row] => require_changed(+ driver::execute_prepared(+ delete_statement,+ vec![Value::from_i64(required_i64(row, 0)?)],+ )?,+ 1,+ ),+ _ => Err(invariant()),+ }+}++fn delete_search_documents_batch(connection: &Arc<Connection>,- entries: &[EncodedRecord],+ profile: SearchProfile,+ keys: &[&RecordKey],) -> Result<(), TursoStorageError> {- let mut delete = driver::prepare(connection, SEARCH_DELETE)?;- let mut upsert = driver::prepare(connection, SEARCH_UPSERT)?;- for entry in entries {- let changed = driver::execute_prepared(- &mut delete,- vec![text(&entry.key.typename), text(&entry.key.id)],- )?;- if changed < 0 {- return Err(invariant());+ for keys in keys.chunks(SEARCH_WRITE_BATCH_SIZE) {+ let lookup_sql = std::iter::repeat_n(SEARCH_ROWID, keys.len())+ .collect::<Vec<_>>()+ .join(" UNION ALL ");+ let mut parameters = Vec::with_capacity(keys.len() * 3);+ for key in keys {+ parameters.push(text(profile.as_str()));+ parameters.push(text(&key.typename));+ parameters.push(text(&key.id));}- for document in &entry.search_documents {- require_changed(- driver::execute_prepared(- &mut upsert,- vec![- text(document.profile.as_str()),- text(&entry.key.typename),- text(&entry.key.id),- text(&document.bucket),- text(&document.search_text),- Value::from_i64(document.timestamp_ms),- text(&document.source_hash),- ],- )?,- 1,- )?;+ let rowids = driver::query(connection, &lookup_sql, parameters)?+ .iter()+ .map(|row| required_i64(row, 0))+ .collect::<Result<BTreeSet<_>, _>>()?;+ if rowids.is_empty() {+ continue;}+ let expected = i64::try_from(rowids.len()).map_err(|_| invariant())?;+ require_changed(+ driver::execute(+ connection,+ &format!(+ "DELETE FROM search_documents WHERE rowid IN ({})",+ sql_placeholders(rowids.len())+ ),+ rowids.into_iter().map(Value::from_i64).collect(),+ )?,+ expected,+ )?;+ }+ Ok(())+}++fn upsert_search_documents_batch(+ connection: &Arc<Connection>,+ documents: &[(&RecordKey, &SearchDocument)],+) -> Result<(), TursoStorageError> {+ for documents in documents.chunks(SEARCH_WRITE_BATCH_SIZE) {+ let values_sql = std::iter::repeat_n(SEARCH_UPSERT_ROW, documents.len())+ .collect::<Vec<_>>()+ .join(", ");+ let mut parameters = Vec::with_capacity(documents.len() * 7);+ for (key, document) in documents {+ parameters.push(text(document.profile.as_str()));+ parameters.push(text(&key.typename));+ parameters.push(text(&key.id));+ parameters.push(text(&document.bucket));+ parameters.push(text(&document.search_text));+ parameters.push(Value::from_i64(document.timestamp_ms));+ parameters.push(text(&document.source_hash));+ }+ require_changed(+ driver::execute(+ connection,+ &format!("{SEARCH_UPSERT_PREFIX}{values_sql}{SEARCH_UPSERT_SUFFIX}"),+ parameters,+ )?,+ i64::try_from(documents.len()).map_err(|_| invariant())?,+ )?;}Ok(())}+fn write_search_documents(+ connection: &Arc<Connection>,+ entries: &[EncodedRecord],+) -> Result<(), TursoStorageError> {+ let mut last_entry_by_key = HashMap::with_capacity(entries.len());+ for (index, entry) in entries.iter().enumerate() {+ last_entry_by_key.insert((entry.key.typename.as_str(), entry.key.id.as_str()), index);+ }+ let entries = entries+ .iter()+ .enumerate()+ .filter_map(|(index, entry)| {+ (last_entry_by_key.get(&(entry.key.typename.as_str(), entry.key.id.as_str()))+ == Some(&index))+ .then_some(entry)+ })+ .collect::<Vec<_>>();+ let stale_keys = entries+ .iter()+ .filter(|entry| {+ !entry+ .search_documents+ .iter()+ .any(|document| document.profile == SearchProfile::QuickAccessV1)+ })+ .map(|entry| &entry.key)+ .collect::<Vec<_>>();+ delete_search_documents_batch(connection, SearchProfile::QuickAccessV1, &stale_keys)?;++ let documents = entries+ .iter()+ .flat_map(|entry| {+ entry+ .search_documents+ .iter()+ .map(|document| (&entry.key, document))+ })+ .collect::<Vec<_>>();+ upsert_search_documents_batch(connection, &documents)+}+fn parse_search_document(row: &[Value],profile: SearchProfile,diff --git a/crates/client/cache-turso/src/storage/test.rs b/crates/client/cache-turso/src/storage/test.rsindex 3e0a68f73c..ec1e47a1e0 100644--- a/crates/client/cache-turso/src/storage/test.rs+++ b/crates/client/cache-turso/src/storage/test.rs@@ -21,6 +21,23 @@ fn record(value: &str) -> Record {record}+fn quick_access_document(name: &str, timestamp: u64) -> Record {+ let mut document = Record::default();+ document.fields.insert(+ "__typename".into(),+ cache_core::value::CacheValue::String("GraphqlSoupDocument".into()),+ );+ document.fields.insert(+ "name".into(),+ cache_core::value::CacheValue::String(name.into()),+ );+ document.fields.insert(+ "updatedAt".into(),+ cache_core::value::CacheValue::Number(cache_core::value::CacheNumber::PosInt(timestamp)),+ );+ document+}+fn queued(label: &str) -> NewQueuedMutation {NewQueuedMutation {uuid: uuid::Uuid::new_v4(),@@ -156,22 +173,10 @@ async fn expect_every_storage_method_latched(}#[test]-fn search_projection_is_write_through_and_recent_query_uses_projection_index() {+fn search_projection_is_write_through_and_queries_use_projection_indexes() {block_on(async {let mut storage = TursoStorage::open_in_memory("search-projection").unwrap();- let mut document = Record::default();- document.fields.insert(- "__typename".into(),- cache_core::value::CacheValue::String("GraphqlSoupDocument".into()),- );- document.fields.insert(- "name".into(),- cache_core::value::CacheValue::String("Quarterly Plan".into()),- );- document.fields.insert(- "updatedAt".into(),- cache_core::value::CacheValue::Number(cache_core::value::CacheNumber::PosInt(123)),- );+ let document = quick_access_document("Quarterly Plan", 123);storage.put_batch(vec![(key("GraphqlSoupDocument:d1"), document.clone()),@@ -248,6 +253,44 @@ fn search_projection_is_write_through_and_recent_query_uses_projection_index() {);assert!(!details.contains("records"));+ let rowid_plan = driver::query(+ &storage.connection(),+ &format!("EXPLAIN QUERY PLAN {SEARCH_ROWID}"),+ vec![+ text(SearchProfile::QuickAccessV1.as_str()),+ text("GraphqlSoupDocument"),+ text("d1"),+ ],+ )+ .unwrap();+ let rowid_details = rowid_plan+ .iter()+ .filter_map(|row| required_text(row, 3).ok())+ .collect::<Vec<_>>()+ .join(" ");+ assert!(+ rowid_details.contains("sqlite_autoindex_search_documents_1")+ && rowid_details.contains("profile=? AND __typename=? AND id=?"),+ "rowid lookup did not use the complete primary-key index: {rowid_details}"+ );+ assert!(+ !rowid_details.contains("SCAN search_documents"),+ "rowid lookup scanned the projection table: {rowid_details}"+ );++ raw_execute(+ &storage,+ SEARCH_UPSERT,+ vec![+ text("future-profile"),+ text("GraphqlSoupDocument"),+ text("d1"),+ text("document"),+ text("future profile row"),+ Value::from_i64(123),+ text("future-profile-hash"),+ ],+ );storage.delete_batch(&[key("GraphqlSoupDocument:d2")]).await@@ -263,6 +306,63 @@ fn search_projection_is_write_through_and_recent_query_uses_projection_index() {.unwrap().is_empty());+ assert_eq!(+ raw_scalar(+ &storage,+ "SELECT COUNT(*) FROM search_documents WHERE profile = 'future-profile' AND __typename = 'GraphqlSoupDocument' AND id = 'd1'",+ ),+ 1+ );+ });+}++#[test]+fn search_projection_batches_large_writes_and_keeps_the_last_duplicate() {+ block_on(async {+ let mut storage = TursoStorage::open_in_memory("search-projection-batches").unwrap();+ let record_count = SEARCH_WRITE_BATCH_SIZE + 2;+ let mut entries = (0..record_count)+ .map(|index| {+ (+ key(&format!("GraphqlSoupDocument:d{index}")),+ quick_access_document(&format!("Document {index}"), index as u64),+ )+ })+ .collect::<Vec<_>>();+ entries.push((key("GraphqlSoupDocument:d0"), record("internal")));++ storage.put_batch(entries).await.unwrap();+ let loaded = storage+ .load_search_documents(SearchProfile::QuickAccessV1)+ .await+ .unwrap();+ assert_eq!(loaded.len(), record_count - 1);+ assert!(+ loaded+ .iter()+ .all(|document| document.record_key.as_ref() != "GraphqlSoupDocument:d0")+ );++ storage+ .put_batch(+ (1..record_count)+ .map(|index| {+ (+ key(&format!("GraphqlSoupDocument:d{index}")),+ record("internal"),+ )+ })+ .collect(),+ )+ .await+ .unwrap();+ assert!(+ storage+ .load_search_documents(SearchProfile::QuickAccessV1)+ .await+ .unwrap()+ .is_empty()+ );});}