Skip to content

Personal AI, On Personal Devices

Python

fix(connectors): sanitize plain-text FTS queries (#1027)

Lucienbc1df2d8a9e1
diff --git a/src/openjarvis/connectors/store.py b/src/openjarvis/connectors/store.py
index bf68a9d5..2b6d6319 100644
--- a/src/openjarvis/connectors/store.py
+++ b/src/openjarvis/connectors/store.py
@@ -10,6 +10,7 @@ Pure Python ``sqlite3`` (no Rust extension required).
from __future__ import annotations
import json
+import re
import sqlite3
import time
import uuid
@@ -352,7 +353,7 @@ class KnowledgeStore(MemoryBackend):
Parameters
----------
- query: Full-text search query.
+ query: Plain-text search query (FTS5 punctuation is handled internally).
top_k: Maximum number of results.
source: Restrict to chunks from this source (e.g. "gmail").
doc_type: Restrict to chunks of this type (e.g. "email").
@@ -360,8 +361,13 @@ class KnowledgeStore(MemoryBackend):
since: Exclude chunks whose timestamp is earlier than this value.
until: Exclude chunks whose timestamp is later than this value.
"""
- if not query.strip():
+ # Callers pass ordinary text, not FTS5 expressions. Punctuation such
+ # as apostrophes, hyphens and question marks is otherwise parsed as
+ # query syntax and can turn a matching search into an empty result.
+ terms = re.findall(r"\w+", query)
+ if not terms:
return []
+ fts_query = " OR ".join(f'"{term}"' for term in terms)
since_str = _to_iso(since) if since is not None else None
until_str = _to_iso(until) if until is not None else None
@@ -408,7 +414,7 @@ class KnowledgeStore(MemoryBackend):
"""
try:
- rows = self._conn.execute(sql, [query] + params + [top_k]).fetchall()
+ rows = self._conn.execute(sql, [fts_query] + params + [top_k]).fetchall()
except sqlite3.OperationalError:
# Malformed FTS query — return empty rather than crash
return []
diff --git a/tests/connectors/test_store.py b/tests/connectors/test_store.py
index eee32275..697cb65c 100644
--- a/tests/connectors/test_store.py
+++ b/tests/connectors/test_store.py
@@ -64,6 +64,28 @@ def test_store_and_retrieve_basic(ks: KnowledgeStore) -> None:
assert results[0].score > 0.0
+ "query",
+ ["Alice's birthday", "When is Alice's birthday?", "Alice-birthday"],
+)
+def test_retrieve_accepts_plain_text_punctuation(
+ ks: KnowledgeStore, query: str
+) -> None:
+ """Natural queries should find matching content without FTS5 syntax rules."""
+ _store(ks, content="Alice's birthday is May 2", source="notes")
+
+ results = ks.retrieve(query)
+
+ assert any("birthday is May 2" in result.content for result in results)
+
+
+def test_retrieve_accepts_symbol_heavy_terms(ks: KnowledgeStore) -> None:
+ _store(ks, content="The C++ migration guide", source="notes")
+
+ assert ks.retrieve("C++")
+ assert ks.retrieve("++") == []
+
+
def test_retrieve_filter_by_source(ks: KnowledgeStore) -> None:
"""retrieve() with source= returns only chunks from that source."""
_store(ks, content="Email about project alpha", source="gmail", doc_type="email")