Skip to content

YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context

fix(db): guard remaining json_each tag-filter sites in list/count_memories (#4252)

Louis Beaumontb3ce2f4d7519
diff --git a/crates/screenpipe-db/src/db/memories.rs b/crates/screenpipe-db/src/db/memories.rs
index d38e2311e..d5ab8d039 100644
--- a/crates/screenpipe-db/src/db/memories.rs
+++ b/crates/screenpipe-db/src/db/memories.rs
@@ -299,7 +299,7 @@ impl DatabaseManager {
// callers pay nothing.
sql.push_str(&format!(
" AND (json_array_length(?9) = 0 OR \
- (SELECT COUNT(DISTINCT je.value) FROM json_each({tags_col}) je \
+ (SELECT COUNT(DISTINCT je.value) FROM json_each(CASE WHEN json_valid({tags_col}) THEN {tags_col} ELSE '[]' END) je \
WHERE je.value IN (SELECT value FROM json_each(?9))) = json_array_length(?9))"
));
@@ -380,7 +380,7 @@ impl DatabaseManager {
}
sql.push_str(&format!(
" AND (json_array_length(?7) = 0 OR \
- (SELECT COUNT(DISTINCT je.value) FROM json_each({tags_col}) je \
+ (SELECT COUNT(DISTINCT je.value) FROM json_each(CASE WHEN json_valid({tags_col}) THEN {tags_col} ELSE '[]' END) je \
WHERE je.value IN (SELECT value FROM json_each(?7))) = json_array_length(?7))"
));
@@ -400,6 +400,10 @@ impl DatabaseManager {
pub async fn list_memory_tags(&self) -> Result<Vec<String>, SqlxError> {
// Tags are stored as JSON arrays. Extract all unique tag values across all memories.
+ // Guard `json_each` against rows whose `tags` isn't valid JSON (e.g. a
+ // legacy/sync row that landed an empty string or plain text): a single
+ // malformed value makes SQLite raise "malformed JSON" and 500s the whole
+ // query. Treat non-JSON as an empty array instead. See `autocomplete_tags`.
let rows: Vec<(String,)> = sqlx::query_as(
"SELECT DISTINCT j.value FROM memories, json_each(CASE WHEN json_valid(memories.tags) THEN memories.tags ELSE '[]' END) j \
WHERE j.value IS NOT NULL AND j.value != '' \
@@ -421,6 +425,12 @@ impl DatabaseManager {
let candidate_limit = (limit + offset).clamp(1, 200);
let search = query.trim();
+ // `json_each(memories.tags)` raises SQLite's "malformed JSON" runtime
+ // error if ANY single memory row carries a `tags` value that isn't a
+ // valid JSON array. Most writers serialize via serde_json, but the
+ // cloud-sync ingest path (`upsert_synced_memory`) binds the remote
+ // `tags` string verbatim, so one bad row would 500 the whole endpoint
+ // and kill tag autocomplete for everyone. Coerce non-JSON to `[]`.
sqlx::query_as::<_, TagAutocompleteItem>(
r#"
WITH candidates AS (
diff --git a/crates/screenpipe-db/tests/tag_autocomplete_query_test.rs b/crates/screenpipe-db/tests/tag_autocomplete_query_test.rs
index 9c049bf35..1751d1b1c 100644
--- a/crates/screenpipe-db/tests/tag_autocomplete_query_test.rs
+++ b/crates/screenpipe-db/tests/tag_autocomplete_query_test.rs
@@ -181,6 +181,48 @@ async fn tag_autocomplete_query_tolerates_malformed_memory_tags() {
assert!(tag_list.iter().any(|t| t == "shared"));
}
+/// The exact-match tag filter in `list_memories` / `count_memories` runs the
+/// SAME `json_each` over `memories.tags`, but only when a `tags` filter is
+/// actually supplied (the `json_array_length(?) = 0` short-circuit skips it
+/// otherwise). A malformed row therefore 500s `GET /memories?tags=...` too —
+/// guard it the same way.
+#[tokio::test]
+async fn memory_tag_filter_tolerates_malformed_memory_tags() {
+ let db = migrated_db().await;
+
+ exec(
+ &db,
+ "INSERT INTO memories (content, tags) VALUES \
+ ('ok1', '[\"keep\"]'), \
+ ('empty', ''), \
+ ('null', NULL), \
+ ('garbage', 'not json at all'), \
+ ('ok2', '[\"keep\",\"other\"]')",
+ )
+ .await;
+
+ let tags_all = vec!["keep".to_string()];
+
+ // Must not error — both helpers `json_each` over every candidate row, so a
+ // single malformed row would raise "malformed JSON" before the guard.
+ let count = db
+ .count_memories(None, None, None, None, None, None, &tags_all)
+ .await
+ .unwrap();
+ assert_eq!(count, 2, "two memories carry the `keep` tag");
+
+ let rows = db
+ .list_memories(
+ None, None, None, None, None, None, 50, 0, None, None, &tags_all,
+ )
+ .await
+ .unwrap();
+ assert_eq!(rows.len(), 2);
+ assert!(rows
+ .iter()
+ .all(|m| m.tags.as_deref().unwrap_or("").contains("keep")));
+}
+
#[tokio::test]
async fn tag_autocomplete_query_survives_large_mixed_sources() {
let db = migrated_db().await;