Personal AI, On Personal Devices
Python
fix(connectors): sanitize plain-text FTS queries (#1027)
Lucienbc1df2d8a9e1
diff --git a/src/openjarvis/connectors/store.py b/src/openjarvis/connectors/store.pyindex bf68a9d5..2b6d6319 100644--- a/src/openjarvis/connectors/store.py+++ b/src/openjarvis/connectors/store.py@@ -10,6 +10,7 @@ Pure Python ``sqlite3`` (no Rust extension required).from __future__ import annotationsimport json+import reimport sqlite3import timeimport uuid@@ -352,7 +353,7 @@ class KnowledgeStore(MemoryBackend):Parameters----------- query: Full-text search query.+ query: Plain-text search query (FTS5 punctuation is handled internally).top_k: Maximum number of results.source: Restrict to chunks from this source (e.g. "gmail").doc_type: Restrict to chunks of this type (e.g. "email").@@ -360,8 +361,13 @@ class KnowledgeStore(MemoryBackend):since: Exclude chunks whose timestamp is earlier than this value.until: Exclude chunks whose timestamp is later than this value."""- if not query.strip():+ # Callers pass ordinary text, not FTS5 expressions. Punctuation such+ # as apostrophes, hyphens and question marks is otherwise parsed as+ # query syntax and can turn a matching search into an empty result.+ terms = re.findall(r"\w+", query)+ if not terms:return []+ fts_query = " OR ".join(f'"{term}"' for term in terms)since_str = _to_iso(since) if since is not None else Noneuntil_str = _to_iso(until) if until is not None else None@@ -408,7 +414,7 @@ class KnowledgeStore(MemoryBackend):"""try:- rows = self._conn.execute(sql, [query] + params + [top_k]).fetchall()+ rows = self._conn.execute(sql, [fts_query] + params + [top_k]).fetchall()except sqlite3.OperationalError:# Malformed FTS query — return empty rather than crashreturn []diff --git a/tests/connectors/test_store.py b/tests/connectors/test_store.pyindex eee32275..697cb65c 100644--- a/tests/connectors/test_store.py+++ b/tests/connectors/test_store.py@@ -64,6 +64,28 @@ def test_store_and_retrieve_basic(ks: KnowledgeStore) -> None:assert results[0].score > 0.0+ "query",+ ["Alice's birthday", "When is Alice's birthday?", "Alice-birthday"],+)+def test_retrieve_accepts_plain_text_punctuation(+ ks: KnowledgeStore, query: str+) -> None:+ """Natural queries should find matching content without FTS5 syntax rules."""+ _store(ks, content="Alice's birthday is May 2", source="notes")++ results = ks.retrieve(query)++ assert any("birthday is May 2" in result.content for result in results)+++def test_retrieve_accepts_symbol_heavy_terms(ks: KnowledgeStore) -> None:+ _store(ks, content="The C++ migration guide", source="notes")++ assert ks.retrieve("C++")+ assert ks.retrieve("++") == []++def test_retrieve_filter_by_source(ks: KnowledgeStore) -> None:"""retrieve() with source= returns only chunks from that source."""_store(ks, content="Email about project alpha", source="gmail", doc_type="email")