YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context
Rust agentsagiaiai-memoryaudio-recordingcomputer-visionhermeshermes-agentllmlocal-ailocal-firstmachine-learningmcpmultimodalopenclawprivacyrewindscreen-recordingspeech-to-textycombinator
fix(db): guard json_each against non-JSON memory tags (fix tag autocomplete 500) (#4253)
Louis Beaumontb0d048f898d1
diff --git a/crates/screenpipe-db/src/db/memories.rs b/crates/screenpipe-db/src/db/memories.rsindex 4b7620f4c..d38e2311e 100644--- a/crates/screenpipe-db/src/db/memories.rs+++ b/crates/screenpipe-db/src/db/memories.rs@@ -401,7 +401,7 @@ impl DatabaseManager {pub async fn list_memory_tags(&self) -> Result<Vec<String>, SqlxError> {// Tags are stored as JSON arrays. Extract all unique tag values across all memories.let rows: Vec<(String,)> = sqlx::query_as(- "SELECT DISTINCT j.value FROM memories, json_each(memories.tags) j \+ "SELECT DISTINCT j.value FROM memories, json_each(CASE WHEN json_valid(memories.tags) THEN memories.tags ELSE '[]' END) j \WHERE j.value IS NOT NULL AND j.value != '' \ORDER BY j.value",)@@ -443,7 +443,7 @@ impl DatabaseManager {SELECT nameFROM (SELECT json_tags.value as name- FROM memories, json_each(memories.tags) json_tags+ FROM memories, json_each(CASE WHEN json_valid(memories.tags) THEN memories.tags ELSE '[]' END) json_tagsWHERE json_tags.value IS NOT NULLAND json_tags.value != ''AND (? = '' OR json_tags.value LIKE '%' || ? || '%' COLLATE NOCASE)@@ -470,7 +470,7 @@ impl DatabaseManager {WHERE t.name = candidates.name) + (SELECT COUNT(DISTINCT memories.id)- FROM memories, json_each(memories.tags) memory_tags+ FROM memories, json_each(CASE WHEN json_valid(memories.tags) THEN memories.tags ELSE '[]' END) memory_tagsWHERE memory_tags.value = candidates.name) as count,(@@ -487,7 +487,7 @@ impl DatabaseManager {) as audio_count,(SELECT COUNT(DISTINCT memories.id)- FROM memories, json_each(memories.tags) memory_tags+ FROM memories, json_each(CASE WHEN json_valid(memories.tags) THEN memories.tags ELSE '[]' END) memory_tagsWHERE memory_tags.value = candidates.name) as memory_countFROM candidatesdiff --git a/crates/screenpipe-db/tests/tag_autocomplete_query_test.rs b/crates/screenpipe-db/tests/tag_autocomplete_query_test.rsindex a573e44ed..9c049bf35 100644--- a/crates/screenpipe-db/tests/tag_autocomplete_query_test.rs+++ b/crates/screenpipe-db/tests/tag_autocomplete_query_test.rs@@ -142,6 +142,45 @@ async fn tag_autocomplete_query_counts_tags_across_screen_audio_and_memories() {assert_eq!(rows["memory-only"].4, 2);}+/// Regression: a memory row whose `tags` column is empty (`''`), NULL, or+/// non-JSON must not blow up the autocomplete query. Before the `json_valid`+/// guard, a single `tags = ''` row made `json_each(memories.tags)` raise+/// "(code: 1) malformed JSON", which 500-ed `GET /tags/autocomplete` for the+/// whole chat tag filter (observed firing ~15k times/day in production).+#[tokio::test]+async fn tag_autocomplete_query_tolerates_malformed_memory_tags() {+ let db = migrated_db().await;++ exec(+ &db,+ "INSERT INTO memories (content, tags) VALUES \+ ('ok1', '[\"shared\",\"memory-only\"]'), \+ ('empty', ''), \+ ('null', NULL), \+ ('garbage', 'not json at all'), \+ ('ok2', '[\"memory-only\"]')",+ )+ .await;++ // Must not error — this is the regression. `.unwrap()` inside `tag_rows`+ // would panic with "malformed JSON" before the guard was added.+ let rows = by_name(tag_rows(&db).await);++ // Valid memory tags still surface with correct counts; bad rows are skipped.+ assert_eq!(rows["memory-only"].4, 2, "memory_count for memory-only");+ assert_eq!(rows["shared"].4, 1, "memory_count for shared");+ assert!(+ !rows.contains_key(""),+ "empty/malformed tags must not surface as a tag"+ );++ // `list_memory_tags` shares the same `json_each(memories.tags)` pattern and+ // must tolerate the bad rows too.+ let tag_list = db.list_memory_tags().await.unwrap();+ assert!(tag_list.iter().any(|t| t == "memory-only"));+ assert!(tag_list.iter().any(|t| t == "shared"));+}+#[tokio::test]async fn tag_autocomplete_query_survives_large_mixed_sources() {let db = migrated_db().await;