Skip to content

"Vibe-Trading: Your Personal Trading Agent"

fix: make cross-session search actually match Chinese, Japanese, and Korean text

Lanre Shittued85d75530c5
diff --git a/agent/src/session/search.py b/agent/src/session/search.py
index bf68ee2d4..cdb98974c 100644
--- a/agent/src/session/search.py
+++ b/agent/src/session/search.py
@@ -20,6 +20,7 @@ from pathlib import Path
from typing import List, Optional
from src.config.paths import get_runtime_root
+from src.memory.search_index import MemorySearchIndex
logger = logging.getLogger(__name__)
@@ -180,10 +181,16 @@ class SessionSearchIndex:
if not content or not content.strip():
return
conn = self._get_conn()
+ # FTS5's default tokenizer has no notion of CJK word boundaries, so a
+ # run of Chinese/Japanese/Korean characters indexes as one token
+ # spanning the whole run. Storing it pre-split into unigrams/bigrams
+ # (the same scheme _sanitize_fts_query queries with) is what makes a
+ # CJK query match anything shorter than the entire message.
+ prepared = MemorySearchIndex._prepare_cjk(content[:50_000])
conn.execute(
"INSERT INTO messages (session_id, role, content, tool_name, timestamp) "
"VALUES (?, ?, ?, ?, ?)",
- (session_id, role, content[:50_000], tool_name, time.time()),
+ (session_id, role, prepared, tool_name, time.time()),
)
conn.execute(
"UPDATE sessions SET message_count = message_count + 1 WHERE id = ?",
@@ -195,7 +202,9 @@ class SessionSearchIndex:
def _sanitize_fts_query(query: str) -> str:
"""Sanitize a user query for FTS5 MATCH syntax.
- - Splits on non-alphanumeric/CJK to extract tokens
+ - Splits on non-alphanumeric/CJK to extract tokens, expanding a run
+ of CJK characters into the same unigram+bigram tokens
+ ``index_message`` stores content as (see ``MemorySearchIndex``)
- Joins with OR so any-word-matches (not all-words-required)
- Quotes each token to prevent FTS5 operator interpretation
@@ -205,13 +214,7 @@ class SessionSearchIndex:
Returns:
FTS5-safe MATCH expression.
"""
- import re as _re
- # Extract alphanumeric tokens (3+ chars) and CJK characters
- tokens = _re.findall(r"[a-zA-Z0-9_]{2,}|[\u4e00-\u9fff\u3400-\u4dbf]", query)
- if not tokens:
- return '""'
- # Quote each token and join with OR for broader matching
- return " OR ".join(f'"{t}"' for t in tokens)
+ return MemorySearchIndex._sanitize_fts_query(query)
def search(self, query: str, max_sessions: int = 3) -> List[SearchMatch]:
"""Full-text search across all sessions.
@@ -258,7 +261,7 @@ class SessionSearchIndex:
title=row[1] or "(untitled)",
started_at=self._format_time(row[2]),
message_count=row[3],
- snippet=row[4],
+ snippet=MemorySearchIndex._clean_cjk(row[4] or ""),
rank=row[5],
)
if len(seen) >= max_sessions:
diff --git a/agent/tests/test_session_search.py b/agent/tests/test_session_search.py
index e4c1ce719..ed2402c08 100644
--- a/agent/tests/test_session_search.py
+++ b/agent/tests/test_session_search.py
@@ -101,11 +101,28 @@ class TestSearch:
def test_cjk_search(self, index: SessionSearchIndex) -> None:
index.index_session("s1", "A股分析")
index.index_message("s1", "user", "上证指数今日走势分析 shanghai composite index")
- # FTS5 tokenizes by whitespace; CJK single chars may not match.
- # Search with the ASCII fallback to verify the session is indexed.
results = index.search("shanghai composite index")
assert len(results) >= 1
+ def test_cjk_search_matches_a_phrase_shorter_than_the_whole_message(
+ self, index: SessionSearchIndex
+ ) -> None:
+ """A CJK query must match without quoting the entire indexed message.
+
+ FTS5's default tokenizer has no CJK word boundaries, so a run of
+ Chinese characters indexes as one token spanning the whole run.
+ Without unigram/bigram preparation, only a query that happens to be
+ that exact full token could ever match -- any realistic phrase or
+ single word came back empty even though the text was indexed.
+ """
+ index.index_session("s1", "A股分析")
+ index.index_message("s1", "user", "我想了解比特币的最新价格走势和交易策略")
+
+ for query in ("比特币", "价格", "交易策略"):
+ results = index.search(query)
+ assert len(results) >= 1, f"query {query!r} found nothing"
+ assert results[0].session_id == "s1"
+
def test_snippet_contains_match(self, index: SessionSearchIndex) -> None:
index.index_session("s1", "Snippet test")
index.index_message("s1", "user", "The quick brown fox jumps over the lazy dog")