Skip to content
Python

feat(memory): add SQLite FTS5 search index and memory_search tool (#616)

Çağdaş Yürekli1a3ecc6c1602
diff --git a/coworker/memory/base.py b/coworker/memory/base.py
index f6c12e09..0593ee6d 100644
--- a/coworker/memory/base.py
+++ b/coworker/memory/base.py
@@ -67,6 +67,16 @@ class MemoryStore(ABC):
@abstractmethod
def delete_all(self, *, scope: Optional[Scope] = None) -> int: ...
+ @abstractmethod
+ def search(
+ self,
+ query: str,
+ *,
+ limit: int = 5,
+ scope: Optional[Scope] = None,
+ workspace: Optional[str] = None,
+ ) -> list[MemoryItem]: ...
+
# MEMORY-SPEC §7: below this rendered size, every memory is injected in full; above it,
# the block flips to index mode (newest few in full, one-line summaries for the rest,
diff --git a/coworker/memory/sqlite_store.py b/coworker/memory/sqlite_store.py
index 4829081e..08e38416 100644
--- a/coworker/memory/sqlite_store.py
+++ b/coworker/memory/sqlite_store.py
@@ -2,6 +2,7 @@
from __future__ import annotations
+import re
import sqlite3
import threading
from pathlib import Path
@@ -42,6 +43,53 @@ class SQLiteMemoryStore(MemoryStore):
self._conn.execute("ALTER TABLE memories ADD COLUMN summary TEXT")
self._conn.commit()
+ # FTS5 full-text search index (companion virtual table + triggers)
+ self._fts_enabled = False
+ try:
+ fts_exists = self._conn.execute(
+ "SELECT 1 FROM sqlite_master WHERE type='table' AND name='memories_fts'"
+ ).fetchone()
+ self._conn.execute("""
+ CREATE VIRTUAL TABLE IF NOT EXISTS memories_fts USING fts5(
+ content,
+ summary,
+ content=memories,
+ content_rowid=id
+ )
+ """)
+ self._conn.execute("""
+ CREATE TRIGGER IF NOT EXISTS memories_ai AFTER INSERT ON memories BEGIN
+ INSERT INTO memories_fts(rowid, content, summary)
+ VALUES (new.id, new.content, new.summary);
+ END;
+ """)
+ self._conn.execute("""
+ CREATE TRIGGER IF NOT EXISTS memories_ad AFTER DELETE ON memories BEGIN
+ INSERT INTO memories_fts(memories_fts, rowid, content, summary)
+ VALUES('delete', old.id, old.content, old.summary);
+ END;
+ """)
+ self._conn.execute("""
+ CREATE TRIGGER IF NOT EXISTS memories_au AFTER UPDATE ON memories BEGIN
+ INSERT INTO memories_fts(memories_fts, rowid, content, summary)
+ VALUES('delete', old.id, old.content, old.summary);
+ INSERT INTO memories_fts(rowid, content, summary)
+ VALUES (new.id, new.content, new.summary);
+ END;
+ """)
+ if not fts_exists:
+ mem_count = self._conn.execute(
+ "SELECT count(*) FROM memories"
+ ).fetchone()[0]
+ if mem_count > 0:
+ self._conn.execute(
+ "INSERT INTO memories_fts(memories_fts) VALUES('rebuild')"
+ )
+ self._conn.commit()
+ self._fts_enabled = True
+ except sqlite3.OperationalError:
+ self._fts_enabled = False
+
def add(
self,
content: str,
@@ -143,6 +191,94 @@ class SQLiteMemoryStore(MemoryStore):
self._conn.commit()
return cursor.rowcount
+ def search(
+ self,
+ query: str,
+ *,
+ limit: int = 5,
+ scope: Optional[Scope] = None,
+ workspace: Optional[str] = None,
+ ) -> list[MemoryItem]:
+ """Search memories using FTS5 full-text index with fallback to LIKE matching."""
+ if not query or not query.strip():
+ return []
+ if not getattr(self, "_fts_enabled", False):
+ return self._search_like(
+ query, limit=limit, scope=scope, workspace=workspace
+ )
+
+ tokens = re.findall(r"\w+", query)
+ if not tokens:
+ return []
+ fts_query = " ".join(f'"{t}"*' for t in tokens)
+
+ sql = (
+ "SELECT m.* FROM memories m "
+ "JOIN memories_fts f ON m.id = f.rowid "
+ "WHERE memories_fts MATCH ?"
+ )
+ params: list[object] = [fts_query]
+ if scope is not None:
+ sql += " AND m.scope = ?"
+ params.append(Scope(scope).value)
+ if workspace is not None:
+ sql += " AND m.workspace = ?"
+ params.append(workspace)
+ elif workspace is not None:
+ sql += (
+ " AND (m.scope = 'global' OR (m.scope = 'workspace' AND m.workspace = ?))"
+ )
+ params.append(workspace)
+ sql += " ORDER BY f.rank LIMIT ?"
+ params.append(limit)
+
+ with self._lock:
+ try:
+ rows = self._conn.execute(sql, params).fetchall()
+ return [_row_to_item(row) for row in rows]
+ except sqlite3.OperationalError:
+ return self._search_like(
+ query, limit=limit, scope=scope, workspace=workspace
+ )
+
+ def _search_like(
+ self,
+ query: str,
+ *,
+ limit: int = 5,
+ scope: Optional[Scope] = None,
+ workspace: Optional[str] = None,
+ ) -> list[MemoryItem]:
+ like_pattern = f"%{query.strip()}%"
+ sql = "SELECT * FROM memories WHERE (content LIKE ? OR summary LIKE ?)"
+ params: list[object] = [like_pattern, like_pattern]
+ if scope is not None:
+ sql += " AND scope = ?"
+ params.append(Scope(scope).value)
+ if workspace is not None:
+ sql += " AND workspace = ?"
+ params.append(workspace)
+ elif workspace is not None:
+ sql += (
+ " AND (scope = 'global' OR (scope = 'workspace' AND workspace = ?))"
+ )
+ params.append(workspace)
+ sql += " ORDER BY id DESC LIMIT ?"
+ params.append(limit)
+ with self._lock:
+ rows = self._conn.execute(sql, params).fetchall()
+ return [_row_to_item(row) for row in rows]
+
+ def rebuild_index(self) -> None:
+ """Rebuild the FTS5 search index."""
+ if not getattr(self, "_fts_enabled", False):
+ return
+ with self._lock:
+ self._conn.execute(
+ "INSERT INTO memories_fts(memories_fts) VALUES('rebuild')"
+ )
+ self._conn.commit()
+
def close(self) -> None:
self._conn.close()
diff --git a/coworker/memory/tools.py b/coworker/memory/tools.py
index 5997f2ef..2a23c9f0 100644
--- a/coworker/memory/tools.py
+++ b/coworker/memory/tools.py
@@ -137,7 +137,27 @@ def memory_tools(
return {"deleted": True, "id": memory_id}
return {"deleted": False, "error": f"no memory with id {memory_id}"}
+ def memory_search(query: str, limit: int = 5) -> dict:
+ """Search memories by keyword or topic.
+
+ Args:
+ query (str): Words or phrase to search for.
+ limit (int): Maximum number of memories to return (default 5).
+ """
+ items = store.search(query, limit=limit, workspace=workspace)
+ return {
+ "memories": [
+ {
+ "id": item.id,
+ "scope": item.scope.value,
+ "content": item.content,
+ "summary": item.summary,
+ }
+ for item in items
+ ]
+ }
+
return [
ai.tool(fn, metadata=ai.ToolMetadata(**_META))
- for fn in (remember, memory_read, memory_update, memory_forget)
+ for fn in (remember, memory_read, memory_update, memory_forget, memory_search)
]
diff --git a/tests/test_memory.py b/tests/test_memory.py
index e52e4369..baad4355 100644
--- a/tests/test_memory.py
+++ b/tests/test_memory.py
@@ -721,3 +721,148 @@ def test_bookkeeping_saves_do_not_bump_recency(tmp_path):
"SELECT updated_at FROM sessions WHERE session_id = 'rec1'"
).fetchone()
assert row["updated_at"] != "2020-01-01 00:00:00"
+
+
+# -- FTS5 search & memory_search tool ------------------------------------------
+
+
+def test_fts5_search_basic_and_prefix(tmp_path):
+ store = _store(tmp_path)
+ store.add(
+ "uses PostgreSQL for all relational storage",
+ scope=Scope.WORKSPACE,
+ workspace="/proj",
+ )
+ store.add(
+ "prefers pytest-cov for test coverage metrics",
+ scope=Scope.WORKSPACE,
+ workspace="/proj",
+ )
+ store.add("never deploy on Friday afternoons", scope=Scope.GLOBAL)
+
+ # Prefix and keyword matching
+ res = store.search("postgres", workspace="/proj")
+ assert len(res) == 1
+ assert "PostgreSQL" in res[0].content
+
+ # Multi-word query
+ res = store.search("pytest coverage", workspace="/proj")
+ assert len(res) == 1
+ assert "pytest-cov" in res[0].content
+
+ # Empty / whitespace search
+ assert store.search("") == []
+ assert store.search(" ") == []
+ assert store.search("nonexistentword123") == []
+
+
+def test_fts5_search_scope_and_workspace_filtering(tmp_path):
+ store = _store(tmp_path)
+ store.add("global rule: always write type annotations", scope=Scope.GLOBAL)
+ store.add("proj A uses FastAPI", scope=Scope.WORKSPACE, workspace="/proj/a")
+ store.add("proj B uses Django", scope=Scope.WORKSPACE, workspace="/proj/b")
+
+ # Workspace search matches global + current workspace, but not another workspace
+ res_a = store.search("uses", workspace="/proj/a")
+ contents_a = [m.content for m in res_a]
+ assert "proj A uses FastAPI" in contents_a
+ assert "proj B uses Django" not in contents_a
+
+ # Explicit scope filtering
+ res_global = store.search("annotations", scope=Scope.GLOBAL)
+ assert len(res_global) == 1 and res_global[0].scope == Scope.GLOBAL
+
+ res_ws_b = store.search("Django", scope=Scope.WORKSPACE, workspace="/proj/b")
+ assert len(res_ws_b) == 1 and res_ws_b[0].workspace == "/proj/b"
+
+
+def test_fts5_search_triggers_sync_on_update_and_delete(tmp_path):
+ store = _store(tmp_path)
+ item = store.add(
+ "uses React 18 for frontend", scope=Scope.GLOBAL, summary="React frontend"
+ )
+ assert len(store.search("React")) == 1
+
+ # Update item
+ store.update(
+ item.id,
+ "migrated to Next.js 15 for frontend",
+ summary="Next.js frontend",
+ )
+ assert len(store.search("React")) == 0
+ assert len(store.search("Next")) == 1
+
+ # Delete item
+ store.delete(item.id)
+ assert len(store.search("Next")) == 0
+
+
+def test_fts5_search_fallback_when_disabled(tmp_path):
+ store = _store(tmp_path)
+ store.add("prefers dark mode in UI", scope=Scope.GLOBAL)
+ # Simulate environment without FTS5
+ store._fts_enabled = False
+
+ res = store.search("dark mode")
+ assert len(res) == 1
+ assert "dark mode" in res[0].content
+
+
+def test_fts5_search_rebuild_index(tmp_path):
+ path = tmp_path / "legacy_for_fts.db"
+ conn = sqlite3.connect(path)
+ conn.execute(
+ """CREATE TABLE memories (
+ id INTEGER PRIMARY KEY AUTOINCREMENT,
+ scope TEXT NOT NULL,
+ key TEXT,
+ content TEXT NOT NULL,
+ summary TEXT,
+ workspace TEXT,
+ session_id TEXT,
+ created_at TEXT DEFAULT CURRENT_TIMESTAMP
+ )"""
+ )
+ conn.execute(
+ "INSERT INTO memories (scope, content, summary) "
+ "VALUES ('global', 'prefers vim keybindings', 'vim mode')"
+ )
+ conn.commit()
+ conn.close()
+
+ # Initializing SQLiteMemoryStore on existing db should rebuild FTS index
+ store = SQLiteMemoryStore(path)
+ res = store.search("vim")
+ assert len(res) == 1
+ assert "vim" in res[0].content
+
+ # Manual rebuild_index works cleanly
+ store.rebuild_index()
+ assert len(store.search("vim")) == 1
+
+
+def test_memory_search_tool(tmp_path):
+ store = _store(tmp_path)
+ reg = ToolRegistry()
+ reg.register_all(memory_tools(store, workspace="/proj"))
+ assert "memory_search" in reg.names()
+
+ reg.execute(
+ "remember",
+ {"content": "use ruff for python linting", "summary": "ruff linter"},
+ )
+ reg.execute(
+ "remember",
+ {"content": "use eslint for typescript", "summary": "eslint"},
+ )
+
+ result = reg.execute("memory_search", {"query": "ruff"})
+ assert "memories" in result
+ assert len(result["memories"]) == 1
+ assert result["memories"][0]["summary"] == "ruff linter"
+ assert "id" in result["memories"][0]
+
+ # Limit check
+ result_all = reg.execute("memory_search", {"query": "use", "limit": 1})
+ assert len(result_all["memories"]) == 1
+