Skip to content

armgit / macro

Public Imported from GitHub
0

Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.

Seanaye/feat/batch upsert with index (#6204)

Sean Aye7b17df1381ff
diff --git a/crates/client/cache-turso/src/storage.rs b/crates/client/cache-turso/src/storage.rs
index ad66e93140..0eaff1da49 100644
--- a/crates/client/cache-turso/src/storage.rs
+++ b/crates/client/cache-turso/src/storage.rs
@@ -91,8 +91,15 @@ const CREATE_SCHEMA: [&str; 28] = [
const RECORD_GET: &str = "SELECT value FROM records WHERE __typename = ?1 AND id = ?2";
const RECORD_UPSERT: &str = "INSERT INTO records (__typename, id, value) VALUES (?1, ?2, ?3) ON CONFLICT (__typename, id) DO UPDATE SET value = excluded.value";
const RECORD_DELETE: &str = "DELETE FROM records WHERE __typename = ?1 AND id = ?2";
-const SEARCH_DELETE: &str = "DELETE FROM search_documents WHERE __typename = ?1 AND id = ?2";
+// Turso otherwise chooses the browse index's profile prefix for a direct composite-key delete.
+// Force a point lookup through the existing primary-key index, then delete by its rowid.
+const SEARCH_ROWID: &str = "SELECT rowid FROM search_documents INDEXED BY sqlite_autoindex_search_documents_1 WHERE profile = ? AND __typename = ? AND id = ?";
+const SEARCH_DELETE: &str = "DELETE FROM search_documents WHERE rowid = ?1";
const SEARCH_UPSERT: &str = "INSERT INTO search_documents (profile, __typename, id, bucket, search_text, timestamp_ms, source_hash) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7) ON CONFLICT (profile, __typename, id) DO UPDATE SET bucket = excluded.bucket, search_text = excluded.search_text, timestamp_ms = excluded.timestamp_ms, source_hash = excluded.source_hash";
+const SEARCH_UPSERT_PREFIX: &str = "INSERT INTO search_documents (profile, __typename, id, bucket, search_text, timestamp_ms, source_hash) VALUES ";
+const SEARCH_UPSERT_ROW: &str = "(?, ?, ?, ?, ?, ?, ?)";
+const SEARCH_UPSERT_SUFFIX: &str = " ON CONFLICT (profile, __typename, id) DO UPDATE SET bucket = excluded.bucket, search_text = excluded.search_text, timestamp_ms = excluded.timestamp_ms, source_hash = excluded.source_hash";
+const SEARCH_WRITE_BATCH_SIZE: usize = 100;
const SEARCH_LOAD: &str = "SELECT __typename, id, bucket, search_text, timestamp_ms, source_hash FROM search_documents WHERE profile = ?1";
const SEARCH_BROWSE: &str = "SELECT __typename, id, bucket, search_text, timestamp_ms, source_hash FROM search_documents INDEXED BY search_documents_browse_idx WHERE profile = ?1 AND bucket = ?2 ORDER BY timestamp_ms DESC, __typename ASC, id ASC LIMIT ?3";
const SEARCH_BROWSE_AFTER: &str = "SELECT __typename, id, bucket, search_text, timestamp_ms, source_hash FROM search_documents INDEXED BY search_documents_browse_idx WHERE profile = ?1 AND bucket = ?2 AND (timestamp_ms < ?3 OR (timestamp_ms = ?3 AND (__typename > ?4 OR (__typename = ?4 AND id > ?5)))) ORDER BY timestamp_ms DESC, __typename ASC, id ASC LIMIT ?6";
@@ -858,7 +865,8 @@ impl Storage for TursoStorage {
let connection = self.connection();
driver::write_transaction(&connection, || {
let mut record_statement = driver::prepare(&connection, RECORD_DELETE)?;
- let mut search_statement = driver::prepare(&connection, SEARCH_DELETE)?;
+ let mut search_rowid_statement = driver::prepare(&connection, SEARCH_ROWID)?;
+ let mut search_delete_statement = driver::prepare(&connection, SEARCH_DELETE)?;
for (index, key) in keys.iter().enumerate() {
let changed = driver::execute_prepared(
&mut record_statement,
@@ -867,13 +875,12 @@ impl Storage for TursoStorage {
if !(0..=1).contains(&changed) {
return Err(invariant());
}
- let search_changed = driver::execute_prepared(
- &mut search_statement,
- vec![text(&key.typename), text(&key.id)],
+ delete_search_document(
+ &mut search_rowid_statement,
+ &mut search_delete_statement,
+ SearchProfile::QuickAccessV1,
+ key,
)?;
- if search_changed < 0 {
- return Err(invariant());
- }
self.fault_after(TestFaultSite::Delete, index)?;
}
Ok(())
@@ -1421,7 +1428,8 @@ impl PredicateIndexStorage for TursoStorage {
let connection = self.connection();
driver::write_transaction(&connection, || {
let mut record_statement = driver::prepare(&connection, RECORD_DELETE)?;
- let mut search_statement = driver::prepare(&connection, SEARCH_DELETE)?;
+ let mut search_rowid_statement = driver::prepare(&connection, SEARCH_ROWID)?;
+ let mut search_delete_statement = driver::prepare(&connection, SEARCH_DELETE)?;
for key in &keys {
let changed = driver::execute_prepared(
&mut record_statement,
@@ -1430,9 +1438,11 @@ impl PredicateIndexStorage for TursoStorage {
if !(0..=1).contains(&changed) {
return Err(invariant());
}
- driver::execute_prepared(
- &mut search_statement,
- vec![text(&key.typename), text(&key.id)],
+ delete_search_document(
+ &mut search_rowid_statement,
+ &mut search_delete_statement,
+ SearchProfile::QuickAccessV1,
+ key,
)?;
}
for key in projection_keys {
@@ -2594,6 +2604,7 @@ fn initialize(
RECORD_GET,
RECORD_UPSERT,
RECORD_DELETE,
+ SEARCH_ROWID,
SEARCH_DELETE,
SEARCH_UPSERT,
SEARCH_LOAD,
@@ -4089,41 +4100,138 @@ fn prepare_records(
.collect()
}
-fn write_search_documents(
+fn delete_search_document(
+ rowid_statement: &mut turso_core::Statement,
+ delete_statement: &mut turso_core::Statement,
+ profile: SearchProfile,
+ key: &RecordKey,
+) -> Result<(), TursoStorageError> {
+ let rows = driver::query_prepared(
+ rowid_statement,
+ vec![text(profile.as_str()), text(&key.typename), text(&key.id)],
+ )?;
+ match rows.as_slice() {
+ [] => Ok(()),
+ [row] => require_changed(
+ driver::execute_prepared(
+ delete_statement,
+ vec![Value::from_i64(required_i64(row, 0)?)],
+ )?,
+ 1,
+ ),
+ _ => Err(invariant()),
+ }
+}
+
+fn delete_search_documents_batch(
connection: &Arc<Connection>,
- entries: &[EncodedRecord],
+ profile: SearchProfile,
+ keys: &[&RecordKey],
) -> Result<(), TursoStorageError> {
- let mut delete = driver::prepare(connection, SEARCH_DELETE)?;
- let mut upsert = driver::prepare(connection, SEARCH_UPSERT)?;
- for entry in entries {
- let changed = driver::execute_prepared(
- &mut delete,
- vec![text(&entry.key.typename), text(&entry.key.id)],
- )?;
- if changed < 0 {
- return Err(invariant());
+ for keys in keys.chunks(SEARCH_WRITE_BATCH_SIZE) {
+ let lookup_sql = std::iter::repeat_n(SEARCH_ROWID, keys.len())
+ .collect::<Vec<_>>()
+ .join(" UNION ALL ");
+ let mut parameters = Vec::with_capacity(keys.len() * 3);
+ for key in keys {
+ parameters.push(text(profile.as_str()));
+ parameters.push(text(&key.typename));
+ parameters.push(text(&key.id));
}
- for document in &entry.search_documents {
- require_changed(
- driver::execute_prepared(
- &mut upsert,
- vec![
- text(document.profile.as_str()),
- text(&entry.key.typename),
- text(&entry.key.id),
- text(&document.bucket),
- text(&document.search_text),
- Value::from_i64(document.timestamp_ms),
- text(&document.source_hash),
- ],
- )?,
- 1,
- )?;
+ let rowids = driver::query(connection, &lookup_sql, parameters)?
+ .iter()
+ .map(|row| required_i64(row, 0))
+ .collect::<Result<BTreeSet<_>, _>>()?;
+ if rowids.is_empty() {
+ continue;
}
+ let expected = i64::try_from(rowids.len()).map_err(|_| invariant())?;
+ require_changed(
+ driver::execute(
+ connection,
+ &format!(
+ "DELETE FROM search_documents WHERE rowid IN ({})",
+ sql_placeholders(rowids.len())
+ ),
+ rowids.into_iter().map(Value::from_i64).collect(),
+ )?,
+ expected,
+ )?;
+ }
+ Ok(())
+}
+
+fn upsert_search_documents_batch(
+ connection: &Arc<Connection>,
+ documents: &[(&RecordKey, &SearchDocument)],
+) -> Result<(), TursoStorageError> {
+ for documents in documents.chunks(SEARCH_WRITE_BATCH_SIZE) {
+ let values_sql = std::iter::repeat_n(SEARCH_UPSERT_ROW, documents.len())
+ .collect::<Vec<_>>()
+ .join(", ");
+ let mut parameters = Vec::with_capacity(documents.len() * 7);
+ for (key, document) in documents {
+ parameters.push(text(document.profile.as_str()));
+ parameters.push(text(&key.typename));
+ parameters.push(text(&key.id));
+ parameters.push(text(&document.bucket));
+ parameters.push(text(&document.search_text));
+ parameters.push(Value::from_i64(document.timestamp_ms));
+ parameters.push(text(&document.source_hash));
+ }
+ require_changed(
+ driver::execute(
+ connection,
+ &format!("{SEARCH_UPSERT_PREFIX}{values_sql}{SEARCH_UPSERT_SUFFIX}"),
+ parameters,
+ )?,
+ i64::try_from(documents.len()).map_err(|_| invariant())?,
+ )?;
}
Ok(())
}
+fn write_search_documents(
+ connection: &Arc<Connection>,
+ entries: &[EncodedRecord],
+) -> Result<(), TursoStorageError> {
+ let mut last_entry_by_key = HashMap::with_capacity(entries.len());
+ for (index, entry) in entries.iter().enumerate() {
+ last_entry_by_key.insert((entry.key.typename.as_str(), entry.key.id.as_str()), index);
+ }
+ let entries = entries
+ .iter()
+ .enumerate()
+ .filter_map(|(index, entry)| {
+ (last_entry_by_key.get(&(entry.key.typename.as_str(), entry.key.id.as_str()))
+ == Some(&index))
+ .then_some(entry)
+ })
+ .collect::<Vec<_>>();
+ let stale_keys = entries
+ .iter()
+ .filter(|entry| {
+ !entry
+ .search_documents
+ .iter()
+ .any(|document| document.profile == SearchProfile::QuickAccessV1)
+ })
+ .map(|entry| &entry.key)
+ .collect::<Vec<_>>();
+ delete_search_documents_batch(connection, SearchProfile::QuickAccessV1, &stale_keys)?;
+
+ let documents = entries
+ .iter()
+ .flat_map(|entry| {
+ entry
+ .search_documents
+ .iter()
+ .map(|document| (&entry.key, document))
+ })
+ .collect::<Vec<_>>();
+ upsert_search_documents_batch(connection, &documents)
+}
+
fn parse_search_document(
row: &[Value],
profile: SearchProfile,
diff --git a/crates/client/cache-turso/src/storage/test.rs b/crates/client/cache-turso/src/storage/test.rs
index 3e0a68f73c..ec1e47a1e0 100644
--- a/crates/client/cache-turso/src/storage/test.rs
+++ b/crates/client/cache-turso/src/storage/test.rs
@@ -21,6 +21,23 @@ fn record(value: &str) -> Record {
record
}
+fn quick_access_document(name: &str, timestamp: u64) -> Record {
+ let mut document = Record::default();
+ document.fields.insert(
+ "__typename".into(),
+ cache_core::value::CacheValue::String("GraphqlSoupDocument".into()),
+ );
+ document.fields.insert(
+ "name".into(),
+ cache_core::value::CacheValue::String(name.into()),
+ );
+ document.fields.insert(
+ "updatedAt".into(),
+ cache_core::value::CacheValue::Number(cache_core::value::CacheNumber::PosInt(timestamp)),
+ );
+ document
+}
+
fn queued(label: &str) -> NewQueuedMutation {
NewQueuedMutation {
uuid: uuid::Uuid::new_v4(),
@@ -156,22 +173,10 @@ async fn expect_every_storage_method_latched(
}
#[test]
-fn search_projection_is_write_through_and_recent_query_uses_projection_index() {
+fn search_projection_is_write_through_and_queries_use_projection_indexes() {
block_on(async {
let mut storage = TursoStorage::open_in_memory("search-projection").unwrap();
- let mut document = Record::default();
- document.fields.insert(
- "__typename".into(),
- cache_core::value::CacheValue::String("GraphqlSoupDocument".into()),
- );
- document.fields.insert(
- "name".into(),
- cache_core::value::CacheValue::String("Quarterly Plan".into()),
- );
- document.fields.insert(
- "updatedAt".into(),
- cache_core::value::CacheValue::Number(cache_core::value::CacheNumber::PosInt(123)),
- );
+ let document = quick_access_document("Quarterly Plan", 123);
storage
.put_batch(vec![
(key("GraphqlSoupDocument:d1"), document.clone()),
@@ -248,6 +253,44 @@ fn search_projection_is_write_through_and_recent_query_uses_projection_index() {
);
assert!(!details.contains("records"));
+ let rowid_plan = driver::query(
+ &storage.connection(),
+ &format!("EXPLAIN QUERY PLAN {SEARCH_ROWID}"),
+ vec![
+ text(SearchProfile::QuickAccessV1.as_str()),
+ text("GraphqlSoupDocument"),
+ text("d1"),
+ ],
+ )
+ .unwrap();
+ let rowid_details = rowid_plan
+ .iter()
+ .filter_map(|row| required_text(row, 3).ok())
+ .collect::<Vec<_>>()
+ .join(" ");
+ assert!(
+ rowid_details.contains("sqlite_autoindex_search_documents_1")
+ && rowid_details.contains("profile=? AND __typename=? AND id=?"),
+ "rowid lookup did not use the complete primary-key index: {rowid_details}"
+ );
+ assert!(
+ !rowid_details.contains("SCAN search_documents"),
+ "rowid lookup scanned the projection table: {rowid_details}"
+ );
+
+ raw_execute(
+ &storage,
+ SEARCH_UPSERT,
+ vec![
+ text("future-profile"),
+ text("GraphqlSoupDocument"),
+ text("d1"),
+ text("document"),
+ text("future profile row"),
+ Value::from_i64(123),
+ text("future-profile-hash"),
+ ],
+ );
storage
.delete_batch(&[key("GraphqlSoupDocument:d2")])
.await
@@ -263,6 +306,63 @@ fn search_projection_is_write_through_and_recent_query_uses_projection_index() {
.unwrap()
.is_empty()
);
+ assert_eq!(
+ raw_scalar(
+ &storage,
+ "SELECT COUNT(*) FROM search_documents WHERE profile = 'future-profile' AND __typename = 'GraphqlSoupDocument' AND id = 'd1'",
+ ),
+ 1
+ );
+ });
+}
+
+#[test]
+fn search_projection_batches_large_writes_and_keeps_the_last_duplicate() {
+ block_on(async {
+ let mut storage = TursoStorage::open_in_memory("search-projection-batches").unwrap();
+ let record_count = SEARCH_WRITE_BATCH_SIZE + 2;
+ let mut entries = (0..record_count)
+ .map(|index| {
+ (
+ key(&format!("GraphqlSoupDocument:d{index}")),
+ quick_access_document(&format!("Document {index}"), index as u64),
+ )
+ })
+ .collect::<Vec<_>>();
+ entries.push((key("GraphqlSoupDocument:d0"), record("internal")));
+
+ storage.put_batch(entries).await.unwrap();
+ let loaded = storage
+ .load_search_documents(SearchProfile::QuickAccessV1)
+ .await
+ .unwrap();
+ assert_eq!(loaded.len(), record_count - 1);
+ assert!(
+ loaded
+ .iter()
+ .all(|document| document.record_key.as_ref() != "GraphqlSoupDocument:d0")
+ );
+
+ storage
+ .put_batch(
+ (1..record_count)
+ .map(|index| {
+ (
+ key(&format!("GraphqlSoupDocument:d{index}")),
+ record("internal"),
+ )
+ })
+ .collect(),
+ )
+ .await
+ .unwrap();
+ assert!(
+ storage
+ .load_search_documents(SearchProfile::QuickAccessV1)
+ .await
+ .unwrap()
+ .is_empty()
+ );
});
}