YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context
Rust agentsagiaiai-memoryaudio-recordingcomputer-visionhermeshermes-agentllmlocal-ailocal-firstmachine-learningmcpmultimodalopenclawprivacyrewindscreen-recordingspeech-to-textycombinator
fix(db): guard remaining json_each tag-filter sites in list/count_memories (#4252)
Louis Beaumontb3ce2f4d7519
diff --git a/crates/screenpipe-db/src/db/memories.rs b/crates/screenpipe-db/src/db/memories.rsindex d38e2311e..d5ab8d039 100644--- a/crates/screenpipe-db/src/db/memories.rs+++ b/crates/screenpipe-db/src/db/memories.rs@@ -299,7 +299,7 @@ impl DatabaseManager {// callers pay nothing.sql.push_str(&format!(" AND (json_array_length(?9) = 0 OR \- (SELECT COUNT(DISTINCT je.value) FROM json_each({tags_col}) je \+ (SELECT COUNT(DISTINCT je.value) FROM json_each(CASE WHEN json_valid({tags_col}) THEN {tags_col} ELSE '[]' END) je \WHERE je.value IN (SELECT value FROM json_each(?9))) = json_array_length(?9))"));@@ -380,7 +380,7 @@ impl DatabaseManager {}sql.push_str(&format!(" AND (json_array_length(?7) = 0 OR \- (SELECT COUNT(DISTINCT je.value) FROM json_each({tags_col}) je \+ (SELECT COUNT(DISTINCT je.value) FROM json_each(CASE WHEN json_valid({tags_col}) THEN {tags_col} ELSE '[]' END) je \WHERE je.value IN (SELECT value FROM json_each(?7))) = json_array_length(?7))"));@@ -400,6 +400,10 @@ impl DatabaseManager {pub async fn list_memory_tags(&self) -> Result<Vec<String>, SqlxError> {// Tags are stored as JSON arrays. Extract all unique tag values across all memories.+ // Guard `json_each` against rows whose `tags` isn't valid JSON (e.g. a+ // legacy/sync row that landed an empty string or plain text): a single+ // malformed value makes SQLite raise "malformed JSON" and 500s the whole+ // query. Treat non-JSON as an empty array instead. See `autocomplete_tags`.let rows: Vec<(String,)> = sqlx::query_as("SELECT DISTINCT j.value FROM memories, json_each(CASE WHEN json_valid(memories.tags) THEN memories.tags ELSE '[]' END) j \WHERE j.value IS NOT NULL AND j.value != '' \@@ -421,6 +425,12 @@ impl DatabaseManager {let candidate_limit = (limit + offset).clamp(1, 200);let search = query.trim();+ // `json_each(memories.tags)` raises SQLite's "malformed JSON" runtime+ // error if ANY single memory row carries a `tags` value that isn't a+ // valid JSON array. Most writers serialize via serde_json, but the+ // cloud-sync ingest path (`upsert_synced_memory`) binds the remote+ // `tags` string verbatim, so one bad row would 500 the whole endpoint+ // and kill tag autocomplete for everyone. Coerce non-JSON to `[]`.sqlx::query_as::<_, TagAutocompleteItem>(r#"WITH candidates AS (diff --git a/crates/screenpipe-db/tests/tag_autocomplete_query_test.rs b/crates/screenpipe-db/tests/tag_autocomplete_query_test.rsindex 9c049bf35..1751d1b1c 100644--- a/crates/screenpipe-db/tests/tag_autocomplete_query_test.rs+++ b/crates/screenpipe-db/tests/tag_autocomplete_query_test.rs@@ -181,6 +181,48 @@ async fn tag_autocomplete_query_tolerates_malformed_memory_tags() {assert!(tag_list.iter().any(|t| t == "shared"));}+/// The exact-match tag filter in `list_memories` / `count_memories` runs the+/// SAME `json_each` over `memories.tags`, but only when a `tags` filter is+/// actually supplied (the `json_array_length(?) = 0` short-circuit skips it+/// otherwise). A malformed row therefore 500s `GET /memories?tags=...` too —+/// guard it the same way.+#[tokio::test]+async fn memory_tag_filter_tolerates_malformed_memory_tags() {+ let db = migrated_db().await;++ exec(+ &db,+ "INSERT INTO memories (content, tags) VALUES \+ ('ok1', '[\"keep\"]'), \+ ('empty', ''), \+ ('null', NULL), \+ ('garbage', 'not json at all'), \+ ('ok2', '[\"keep\",\"other\"]')",+ )+ .await;++ let tags_all = vec!["keep".to_string()];++ // Must not error — both helpers `json_each` over every candidate row, so a+ // single malformed row would raise "malformed JSON" before the guard.+ let count = db+ .count_memories(None, None, None, None, None, None, &tags_all)+ .await+ .unwrap();+ assert_eq!(count, 2, "two memories carry the `keep` tag");++ let rows = db+ .list_memories(+ None, None, None, None, None, None, 50, 0, None, None, &tags_all,+ )+ .await+ .unwrap();+ assert_eq!(rows.len(), 2);+ assert!(rows+ .iter()+ .all(|m| m.tags.as_deref().unwrap_or("").contains("keep")));+}+#[tokio::test]async fn tag_autocomplete_query_survives_large_mixed_sources() {let db = migrated_db().await;