Python
feat(memory): add SQLite FTS5 search index and memory_search tool (#616)
Çağdaş Yürekli1a3ecc6c1602
diff --git a/coworker/memory/base.py b/coworker/memory/base.pyindex f6c12e09..0593ee6d 100644--- a/coworker/memory/base.py+++ b/coworker/memory/base.py@@ -67,6 +67,16 @@ class MemoryStore(ABC):@abstractmethoddef delete_all(self, *, scope: Optional[Scope] = None) -> int: ...+ @abstractmethod+ def search(+ self,+ query: str,+ *,+ limit: int = 5,+ scope: Optional[Scope] = None,+ workspace: Optional[str] = None,+ ) -> list[MemoryItem]: ...+# MEMORY-SPEC §7: below this rendered size, every memory is injected in full; above it,# the block flips to index mode (newest few in full, one-line summaries for the rest,diff --git a/coworker/memory/sqlite_store.py b/coworker/memory/sqlite_store.pyindex 4829081e..08e38416 100644--- a/coworker/memory/sqlite_store.py+++ b/coworker/memory/sqlite_store.py@@ -2,6 +2,7 @@from __future__ import annotations+import reimport sqlite3import threadingfrom pathlib import Path@@ -42,6 +43,53 @@ class SQLiteMemoryStore(MemoryStore):self._conn.execute("ALTER TABLE memories ADD COLUMN summary TEXT")self._conn.commit()+ # FTS5 full-text search index (companion virtual table + triggers)+ self._fts_enabled = False+ try:+ fts_exists = self._conn.execute(+ "SELECT 1 FROM sqlite_master WHERE type='table' AND name='memories_fts'"+ ).fetchone()+ self._conn.execute("""+ CREATE VIRTUAL TABLE IF NOT EXISTS memories_fts USING fts5(+ content,+ summary,+ content=memories,+ content_rowid=id+ )+ """)+ self._conn.execute("""+ CREATE TRIGGER IF NOT EXISTS memories_ai AFTER INSERT ON memories BEGIN+ INSERT INTO memories_fts(rowid, content, summary)+ VALUES (new.id, new.content, new.summary);+ END;+ """)+ self._conn.execute("""+ CREATE TRIGGER IF NOT EXISTS memories_ad AFTER DELETE ON memories BEGIN+ INSERT INTO memories_fts(memories_fts, rowid, content, summary)+ VALUES('delete', old.id, old.content, old.summary);+ END;+ """)+ self._conn.execute("""+ CREATE TRIGGER IF NOT EXISTS memories_au AFTER UPDATE ON memories BEGIN+ INSERT INTO memories_fts(memories_fts, rowid, content, summary)+ VALUES('delete', old.id, old.content, old.summary);+ INSERT INTO memories_fts(rowid, content, summary)+ VALUES (new.id, new.content, new.summary);+ END;+ """)+ if not fts_exists:+ mem_count = self._conn.execute(+ "SELECT count(*) FROM memories"+ ).fetchone()[0]+ if mem_count > 0:+ self._conn.execute(+ "INSERT INTO memories_fts(memories_fts) VALUES('rebuild')"+ )+ self._conn.commit()+ self._fts_enabled = True+ except sqlite3.OperationalError:+ self._fts_enabled = False+def add(self,content: str,@@ -143,6 +191,94 @@ class SQLiteMemoryStore(MemoryStore):self._conn.commit()return cursor.rowcount+ def search(+ self,+ query: str,+ *,+ limit: int = 5,+ scope: Optional[Scope] = None,+ workspace: Optional[str] = None,+ ) -> list[MemoryItem]:+ """Search memories using FTS5 full-text index with fallback to LIKE matching."""+ if not query or not query.strip():+ return []+ if not getattr(self, "_fts_enabled", False):+ return self._search_like(+ query, limit=limit, scope=scope, workspace=workspace+ )++ tokens = re.findall(r"\w+", query)+ if not tokens:+ return []+ fts_query = " ".join(f'"{t}"*' for t in tokens)++ sql = (+ "SELECT m.* FROM memories m "+ "JOIN memories_fts f ON m.id = f.rowid "+ "WHERE memories_fts MATCH ?"+ )+ params: list[object] = [fts_query]+ if scope is not None:+ sql += " AND m.scope = ?"+ params.append(Scope(scope).value)+ if workspace is not None:+ sql += " AND m.workspace = ?"+ params.append(workspace)+ elif workspace is not None:+ sql += (+ " AND (m.scope = 'global' OR (m.scope = 'workspace' AND m.workspace = ?))"+ )+ params.append(workspace)+ sql += " ORDER BY f.rank LIMIT ?"+ params.append(limit)++ with self._lock:+ try:+ rows = self._conn.execute(sql, params).fetchall()+ return [_row_to_item(row) for row in rows]+ except sqlite3.OperationalError:+ return self._search_like(+ query, limit=limit, scope=scope, workspace=workspace+ )++ def _search_like(+ self,+ query: str,+ *,+ limit: int = 5,+ scope: Optional[Scope] = None,+ workspace: Optional[str] = None,+ ) -> list[MemoryItem]:+ like_pattern = f"%{query.strip()}%"+ sql = "SELECT * FROM memories WHERE (content LIKE ? OR summary LIKE ?)"+ params: list[object] = [like_pattern, like_pattern]+ if scope is not None:+ sql += " AND scope = ?"+ params.append(Scope(scope).value)+ if workspace is not None:+ sql += " AND workspace = ?"+ params.append(workspace)+ elif workspace is not None:+ sql += (+ " AND (scope = 'global' OR (scope = 'workspace' AND workspace = ?))"+ )+ params.append(workspace)+ sql += " ORDER BY id DESC LIMIT ?"+ params.append(limit)+ with self._lock:+ rows = self._conn.execute(sql, params).fetchall()+ return [_row_to_item(row) for row in rows]++ def rebuild_index(self) -> None:+ """Rebuild the FTS5 search index."""+ if not getattr(self, "_fts_enabled", False):+ return+ with self._lock:+ self._conn.execute(+ "INSERT INTO memories_fts(memories_fts) VALUES('rebuild')"+ )+ self._conn.commit()+def close(self) -> None:self._conn.close()diff --git a/coworker/memory/tools.py b/coworker/memory/tools.pyindex 5997f2ef..2a23c9f0 100644--- a/coworker/memory/tools.py+++ b/coworker/memory/tools.py@@ -137,7 +137,27 @@ def memory_tools(return {"deleted": True, "id": memory_id}return {"deleted": False, "error": f"no memory with id {memory_id}"}+ def memory_search(query: str, limit: int = 5) -> dict:+ """Search memories by keyword or topic.++ Args:+ query (str): Words or phrase to search for.+ limit (int): Maximum number of memories to return (default 5).+ """+ items = store.search(query, limit=limit, workspace=workspace)+ return {+ "memories": [+ {+ "id": item.id,+ "scope": item.scope.value,+ "content": item.content,+ "summary": item.summary,+ }+ for item in items+ ]+ }+return [ai.tool(fn, metadata=ai.ToolMetadata(**_META))- for fn in (remember, memory_read, memory_update, memory_forget)+ for fn in (remember, memory_read, memory_update, memory_forget, memory_search)]diff --git a/tests/test_memory.py b/tests/test_memory.pyindex e52e4369..baad4355 100644--- a/tests/test_memory.py+++ b/tests/test_memory.py@@ -721,3 +721,148 @@ def test_bookkeeping_saves_do_not_bump_recency(tmp_path):"SELECT updated_at FROM sessions WHERE session_id = 'rec1'").fetchone()assert row["updated_at"] != "2020-01-01 00:00:00"+++# -- FTS5 search & memory_search tool ------------------------------------------+++def test_fts5_search_basic_and_prefix(tmp_path):+ store = _store(tmp_path)+ store.add(+ "uses PostgreSQL for all relational storage",+ scope=Scope.WORKSPACE,+ workspace="/proj",+ )+ store.add(+ "prefers pytest-cov for test coverage metrics",+ scope=Scope.WORKSPACE,+ workspace="/proj",+ )+ store.add("never deploy on Friday afternoons", scope=Scope.GLOBAL)++ # Prefix and keyword matching+ res = store.search("postgres", workspace="/proj")+ assert len(res) == 1+ assert "PostgreSQL" in res[0].content++ # Multi-word query+ res = store.search("pytest coverage", workspace="/proj")+ assert len(res) == 1+ assert "pytest-cov" in res[0].content++ # Empty / whitespace search+ assert store.search("") == []+ assert store.search(" ") == []+ assert store.search("nonexistentword123") == []+++def test_fts5_search_scope_and_workspace_filtering(tmp_path):+ store = _store(tmp_path)+ store.add("global rule: always write type annotations", scope=Scope.GLOBAL)+ store.add("proj A uses FastAPI", scope=Scope.WORKSPACE, workspace="/proj/a")+ store.add("proj B uses Django", scope=Scope.WORKSPACE, workspace="/proj/b")++ # Workspace search matches global + current workspace, but not another workspace+ res_a = store.search("uses", workspace="/proj/a")+ contents_a = [m.content for m in res_a]+ assert "proj A uses FastAPI" in contents_a+ assert "proj B uses Django" not in contents_a++ # Explicit scope filtering+ res_global = store.search("annotations", scope=Scope.GLOBAL)+ assert len(res_global) == 1 and res_global[0].scope == Scope.GLOBAL++ res_ws_b = store.search("Django", scope=Scope.WORKSPACE, workspace="/proj/b")+ assert len(res_ws_b) == 1 and res_ws_b[0].workspace == "/proj/b"+++def test_fts5_search_triggers_sync_on_update_and_delete(tmp_path):+ store = _store(tmp_path)+ item = store.add(+ "uses React 18 for frontend", scope=Scope.GLOBAL, summary="React frontend"+ )+ assert len(store.search("React")) == 1++ # Update item+ store.update(+ item.id,+ "migrated to Next.js 15 for frontend",+ summary="Next.js frontend",+ )+ assert len(store.search("React")) == 0+ assert len(store.search("Next")) == 1++ # Delete item+ store.delete(item.id)+ assert len(store.search("Next")) == 0+++def test_fts5_search_fallback_when_disabled(tmp_path):+ store = _store(tmp_path)+ store.add("prefers dark mode in UI", scope=Scope.GLOBAL)+ # Simulate environment without FTS5+ store._fts_enabled = False++ res = store.search("dark mode")+ assert len(res) == 1+ assert "dark mode" in res[0].content+++def test_fts5_search_rebuild_index(tmp_path):+ path = tmp_path / "legacy_for_fts.db"+ conn = sqlite3.connect(path)+ conn.execute(+ """CREATE TABLE memories (+ id INTEGER PRIMARY KEY AUTOINCREMENT,+ scope TEXT NOT NULL,+ key TEXT,+ content TEXT NOT NULL,+ summary TEXT,+ workspace TEXT,+ session_id TEXT,+ created_at TEXT DEFAULT CURRENT_TIMESTAMP+ )"""+ )+ conn.execute(+ "INSERT INTO memories (scope, content, summary) "+ "VALUES ('global', 'prefers vim keybindings', 'vim mode')"+ )+ conn.commit()+ conn.close()++ # Initializing SQLiteMemoryStore on existing db should rebuild FTS index+ store = SQLiteMemoryStore(path)+ res = store.search("vim")+ assert len(res) == 1+ assert "vim" in res[0].content++ # Manual rebuild_index works cleanly+ store.rebuild_index()+ assert len(store.search("vim")) == 1+++def test_memory_search_tool(tmp_path):+ store = _store(tmp_path)+ reg = ToolRegistry()+ reg.register_all(memory_tools(store, workspace="/proj"))+ assert "memory_search" in reg.names()++ reg.execute(+ "remember",+ {"content": "use ruff for python linting", "summary": "ruff linter"},+ )+ reg.execute(+ "remember",+ {"content": "use eslint for typescript", "summary": "eslint"},+ )++ result = reg.execute("memory_search", {"query": "ruff"})+ assert "memories" in result+ assert len(result["memories"]) == 1+ assert result["memories"][0]["summary"] == "ruff linter"+ assert "id" in result["memories"][0]++ # Limit check+ result_all = reg.execute("memory_search", {"query": "use", "limit": 1})+ assert len(result_all["memories"]) == 1+