Skip to content

Personal AI, On Personal Devices

Python

feat: add KnowledgeStore with source-aware SQLite schema and filtered BM25 retrieval

krypticmouse8d3505f8f34e
diff --git a/src/openjarvis/connectors/__init__.py b/src/openjarvis/connectors/__init__.py
index 1642e5bd..a48d0d71 100644
--- a/src/openjarvis/connectors/__init__.py
+++ b/src/openjarvis/connectors/__init__.py
@@ -6,5 +6,6 @@ from openjarvis.connectors._stubs import (
Document,
SyncStatus,
)
+from openjarvis.connectors.store import KnowledgeStore
-__all__ = ["Attachment", "BaseConnector", "Document", "SyncStatus"]
+__all__ = ["Attachment", "BaseConnector", "Document", "KnowledgeStore", "SyncStatus"]
diff --git a/src/openjarvis/connectors/store.py b/src/openjarvis/connectors/store.py
new file mode 100644
index 00000000..9e65ff2d
--- /dev/null
+++ b/src/openjarvis/connectors/store.py
@@ -0,0 +1,371 @@
+"""KnowledgeStore — source-aware SQLite/FTS5 memory backend for Deep Research.
+
+Extends ``MemoryBackend`` with per-document provenance columns so that the
+IngestionPipeline and the ``knowledge_search`` tool can filter results by
+source, doc_type, author, and timestamp ranges.
+
+Pure Python ``sqlite3`` (no Rust extension required).
+"""
+
+from __future__ import annotations
+
+import json
+import sqlite3
+import time
+import uuid
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any, Dict, List, Optional, Union
+
+from openjarvis.core.events import EventType, get_event_bus
+from openjarvis.core.registry import MemoryRegistry
+from openjarvis.tools.storage._stubs import MemoryBackend, RetrievalResult
+
+# ---------------------------------------------------------------------------
+# DDL
+# ---------------------------------------------------------------------------
+
+_CREATE_MAIN_TABLE = """
+CREATE TABLE IF NOT EXISTS knowledge_chunks (
+ id TEXT PRIMARY KEY,
+ doc_id TEXT NOT NULL,
+ content TEXT NOT NULL,
+ source TEXT NOT NULL DEFAULT '',
+ doc_type TEXT NOT NULL DEFAULT '',
+ title TEXT NOT NULL DEFAULT '',
+ author TEXT NOT NULL DEFAULT '',
+ participants TEXT NOT NULL DEFAULT '[]',
+ timestamp TEXT NOT NULL DEFAULT '',
+ thread_id TEXT NOT NULL DEFAULT '',
+ url TEXT NOT NULL DEFAULT '',
+ metadata TEXT NOT NULL DEFAULT '{}',
+ chunk_index INTEGER NOT NULL DEFAULT 0,
+ created_at REAL NOT NULL
+);
+"""
+
+_CREATE_FTS_TABLE = """
+CREATE VIRTUAL TABLE IF NOT EXISTS knowledge_fts
+USING fts5(
+ content,
+ title,
+ author,
+ content='knowledge_chunks',
+ content_rowid='rowid',
+ tokenize='porter unicode61'
+);
+"""
+
+_CREATE_TRIGGERS = """
+CREATE TRIGGER IF NOT EXISTS knowledge_ai AFTER INSERT ON knowledge_chunks BEGIN
+ INSERT INTO knowledge_fts(rowid, content, title, author)
+ VALUES (new.rowid, new.content, new.title, new.author);
+END;
+
+CREATE TRIGGER IF NOT EXISTS knowledge_ad AFTER DELETE ON knowledge_chunks BEGIN
+ INSERT INTO knowledge_fts(knowledge_fts, rowid, content, title, author)
+ VALUES ('delete', old.rowid, old.content, old.title, old.author);
+END;
+
+CREATE TRIGGER IF NOT EXISTS knowledge_au AFTER UPDATE ON knowledge_chunks BEGIN
+ INSERT INTO knowledge_fts(knowledge_fts, rowid, content, title, author)
+ VALUES ('delete', old.rowid, old.content, old.title, old.author);
+ INSERT INTO knowledge_fts(rowid, content, title, author)
+ VALUES (new.rowid, new.content, new.title, new.author);
+END;
+"""
+
+_CREATE_INDEXES = """
+CREATE INDEX IF NOT EXISTS idx_kc_source ON knowledge_chunks(source);
+CREATE INDEX IF NOT EXISTS idx_kc_doc_type ON knowledge_chunks(doc_type);
+CREATE INDEX IF NOT EXISTS idx_kc_author ON knowledge_chunks(author);
+CREATE INDEX IF NOT EXISTS idx_kc_timestamp ON knowledge_chunks(timestamp);
+CREATE INDEX IF NOT EXISTS idx_kc_thread_id ON knowledge_chunks(thread_id);
+CREATE INDEX IF NOT EXISTS idx_kc_doc_id ON knowledge_chunks(doc_id);
+"""
+
+# ---------------------------------------------------------------------------
+# Helper
+# ---------------------------------------------------------------------------
+
+
+def _to_iso(ts: Optional[Union[datetime, str]]) -> str:
+ """Normalise a timestamp to ISO 8601 string (UTC)."""
+ if ts is None:
+ return ""
+ if isinstance(ts, str):
+ return ts
+ if ts.tzinfo is None:
+ ts = ts.replace(tzinfo=timezone.utc)
+ return ts.isoformat()
+
+
+# ---------------------------------------------------------------------------
+# KnowledgeStore
+# ---------------------------------------------------------------------------
+
+
[email protected]("knowledge")
+class KnowledgeStore(MemoryBackend):
+ """Source-aware SQLite/FTS5 knowledge store for Deep Research.
+
+ Stores document chunks with rich provenance metadata and supports
+ filtered BM25 retrieval by source, doc_type, author, and timestamp.
+ """
+
+ backend_id: str = "knowledge"
+
+ def __init__(self, db_path: Union[str, Path] = "") -> None:
+ if not db_path:
+ from openjarvis.core.config import DEFAULT_CONFIG_DIR
+ db_path = DEFAULT_CONFIG_DIR / "knowledge.db"
+
+ self._db_path = str(db_path)
+ # Ensure the parent directory exists (skip for :memory:)
+ if self._db_path != ":memory:":
+ Path(self._db_path).parent.mkdir(parents=True, exist_ok=True)
+
+ self._conn = sqlite3.connect(self._db_path, check_same_thread=False)
+ self._conn.row_factory = sqlite3.Row
+ self._setup()
+
+ # ------------------------------------------------------------------
+ # Internal setup
+ # ------------------------------------------------------------------
+
+ def _setup(self) -> None:
+ """Create tables, FTS virtual table, triggers and indexes."""
+ self._conn.execute("PRAGMA journal_mode=WAL;")
+ self._conn.execute("PRAGMA foreign_keys=ON;")
+ self._conn.executescript(
+ _CREATE_MAIN_TABLE
+ + _CREATE_FTS_TABLE
+ + _CREATE_TRIGGERS
+ + _CREATE_INDEXES
+ )
+ self._conn.commit()
+
+ # ------------------------------------------------------------------
+ # MemoryBackend interface
+ # ------------------------------------------------------------------
+
+ def store( # type: ignore[override]
+ self,
+ content: str,
+ *,
+ source: str = "",
+ doc_type: str = "",
+ doc_id: Optional[str] = None,
+ title: str = "",
+ author: str = "",
+ participants: Optional[List[str]] = None,
+ timestamp: Optional[Union[datetime, str]] = None,
+ thread_id: Optional[str] = None,
+ url: Optional[str] = None,
+ metadata: Optional[Dict[str, Any]] = None,
+ chunk_index: int = 0,
+ ) -> str:
+ """Persist a content chunk and return its unique chunk id.
+
+ All source-level fields are merged into the stored metadata so that
+ ``retrieve()`` results carry full provenance.
+ """
+ chunk_id = str(uuid.uuid4())
+ if doc_id is None:
+ doc_id = str(uuid.uuid4())
+
+ ts_str = _to_iso(timestamp)
+ participants_json = json.dumps(participants or [])
+
+ # Merge provenance fields into metadata for easy access in results
+ combined_meta: Dict[str, Any] = dict(metadata or {})
+ combined_meta["source"] = source
+ combined_meta["doc_type"] = doc_type
+ combined_meta["doc_id"] = doc_id
+ combined_meta["title"] = title
+ combined_meta["author"] = author
+ combined_meta["participants"] = participants or []
+ combined_meta["timestamp"] = ts_str
+ combined_meta["thread_id"] = thread_id or ""
+ combined_meta["url"] = url or ""
+ combined_meta["chunk_index"] = chunk_index
+
+ meta_json = json.dumps(combined_meta)
+
+ self._conn.execute(
+ """
+ INSERT INTO knowledge_chunks
+ (id, doc_id, content, source, doc_type, title, author,
+ participants, timestamp, thread_id, url, metadata,
+ chunk_index, created_at)
+ VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+ """,
+ (
+ chunk_id,
+ doc_id,
+ content,
+ source,
+ doc_type,
+ title,
+ author,
+ participants_json,
+ ts_str,
+ thread_id or "",
+ url or "",
+ meta_json,
+ chunk_index,
+ time.time(),
+ ),
+ )
+ self._conn.commit()
+
+ get_event_bus().publish(
+ EventType.MEMORY_STORE,
+ {
+ "backend": self.backend_id,
+ "chunk_id": chunk_id,
+ "doc_id": doc_id,
+ "source": source,
+ "doc_type": doc_type,
+ },
+ )
+ return chunk_id
+
+ def retrieve(
+ self,
+ query: str,
+ *,
+ top_k: int = 5,
+ source: Optional[str] = None,
+ doc_type: Optional[str] = None,
+ author: Optional[str] = None,
+ since: Optional[Union[datetime, str]] = None,
+ until: Optional[Union[datetime, str]] = None,
+ **kwargs: Any,
+ ) -> List[RetrievalResult]:
+ """Search using FTS5 BM25 with optional column filters.
+
+ Parameters
+ ----------
+ query: Full-text search query.
+ top_k: Maximum number of results.
+ source: Restrict to chunks from this source (e.g. "gmail").
+ doc_type: Restrict to chunks of this type (e.g. "email").
+ author: Restrict to chunks authored by this person.
+ since: Exclude chunks whose timestamp is earlier than this value.
+ until: Exclude chunks whose timestamp is later than this value.
+ """
+ if not query.strip():
+ return []
+
+ since_str = _to_iso(since) if since is not None else None
+ until_str = _to_iso(until) if until is not None else None
+
+ # Build the WHERE clause for filter columns
+ filters: List[str] = []
+ params: List[Any] = []
+
+ if source is not None:
+ filters.append("kc.source = ?")
+ params.append(source)
+ if doc_type is not None:
+ filters.append("kc.doc_type = ?")
+ params.append(doc_type)
+ if author is not None:
+ filters.append("kc.author = ?")
+ params.append(author)
+ if since_str:
+ filters.append("kc.timestamp >= ?")
+ params.append(since_str)
+ if until_str:
+ filters.append("kc.timestamp <= ?")
+ params.append(until_str)
+
+ where_clause = ""
+ if filters:
+ where_clause = "AND " + " AND ".join(filters)
+
+ # FTS5 bm25() returns negative scores; abs() gives a positive rank
+ sql = f"""
+ SELECT
+ kc.id,
+ kc.content,
+ kc.source,
+ kc.metadata,
+ abs(bm25(knowledge_fts)) AS score
+ FROM knowledge_fts
+ JOIN knowledge_chunks kc ON knowledge_fts.rowid = kc.rowid
+ WHERE knowledge_fts MATCH ?
+ {where_clause}
+ ORDER BY score DESC
+ LIMIT ?
+ """
+
+ try:
+ rows = self._conn.execute(sql, [query] + params + [top_k]).fetchall()
+ except sqlite3.OperationalError:
+ # Malformed FTS query — return empty rather than crash
+ return []
+
+ results: List[RetrievalResult] = []
+ for row in rows:
+ meta = json.loads(row["metadata"]) if row["metadata"] else {}
+ results.append(
+ RetrievalResult(
+ content=row["content"],
+ score=float(row["score"]),
+ source=row["source"],
+ metadata=meta,
+ )
+ )
+
+ get_event_bus().publish(
+ EventType.MEMORY_RETRIEVE,
+ {
+ "backend": self.backend_id,
+ "query": query,
+ "num_results": len(results),
+ "filters": {
+ "source": source,
+ "doc_type": doc_type,
+ "author": author,
+ "since": since_str,
+ "until": until_str,
+ },
+ },
+ )
+ return results
+
+ def delete(self, doc_id: str) -> bool:
+ """Delete all chunks with the given *doc_id*. Returns True if any existed."""
+ cur = self._conn.execute(
+ "DELETE FROM knowledge_chunks WHERE doc_id = ?", (doc_id,)
+ )
+ self._conn.commit()
+ return cur.rowcount > 0
+
+ def clear(self) -> None:
+ """Remove all stored chunks."""
+ self._conn.executescript(
+ "DELETE FROM knowledge_chunks; DELETE FROM knowledge_fts;"
+ )
+ self._conn.commit()
+
+ # ------------------------------------------------------------------
+ # Extra helpers
+ # ------------------------------------------------------------------
+
+ def count(self) -> int:
+ """Return the total number of stored chunks."""
+ row = self._conn.execute("SELECT COUNT(*) FROM knowledge_chunks").fetchone()
+ return row[0] if row else 0
+
+ def close(self) -> None:
+ """Close the underlying SQLite connection."""
+ try:
+ self._conn.close()
+ except Exception:
+ pass
+
+
+__all__ = ["KnowledgeStore"]
diff --git a/tests/connectors/test_store.py b/tests/connectors/test_store.py
new file mode 100644
index 00000000..90700dfe
--- /dev/null
+++ b/tests/connectors/test_store.py
@@ -0,0 +1,271 @@
+"""Tests for KnowledgeStore — source-aware SQLite/FTS5 memory backend."""
+
+from __future__ import annotations
+
+import time
+from datetime import datetime, timezone
+from pathlib import Path
+
+import pytest
+
+from openjarvis.connectors.store import KnowledgeStore
+
+
+# ---------------------------------------------------------------------------
+# Helpers
+# ---------------------------------------------------------------------------
+
+
+def _store(ks: KnowledgeStore, **kwargs) -> str: # type: ignore[type-arg]
+ """Convenience wrapper with sensible defaults."""
+ defaults = {
+ "content": "default content",
+ "source": "test",
+ "doc_type": "note",
+ "doc_id": None,
+ "title": "",
+ "author": "",
+ "participants": None,
+ "timestamp": None,
+ "thread_id": None,
+ "url": None,
+ "metadata": None,
+ "chunk_index": 0,
+ }
+ defaults.update(kwargs)
+ return ks.store(**defaults) # type: ignore[call-arg]
+
+
+# ---------------------------------------------------------------------------
+# Fixtures
+# ---------------------------------------------------------------------------
+
+
+def ks(tmp_path: Path) -> KnowledgeStore:
+ db = tmp_path / "test_knowledge.db"
+ return KnowledgeStore(db_path=db)
+
+
+# ---------------------------------------------------------------------------
+# Tests
+# ---------------------------------------------------------------------------
+
+
+def test_store_and_retrieve_basic(ks: KnowledgeStore) -> None:
+ """Stored content can be retrieved by a matching query."""
+ _store(ks, content="The quick brown fox jumps over the lazy dog", source="notes", doc_type="note")
+ results = ks.retrieve("quick brown fox", top_k=5)
+ assert len(results) >= 1
+ assert "fox" in results[0].content.lower() or "quick" in results[0].content.lower()
+ assert results[0].score > 0.0
+
+
+def test_retrieve_filter_by_source(ks: KnowledgeStore) -> None:
+ """retrieve() with source= returns only chunks from that source."""
+ _store(ks, content="Email about project alpha", source="gmail", doc_type="email")
+ _store(ks, content="Note about project alpha progress", source="obsidian", doc_type="note")
+
+ gmail_results = ks.retrieve("project alpha", top_k=10, source="gmail")
+ for r in gmail_results:
+ assert r.metadata.get("source") == "gmail"
+
+ obsidian_results = ks.retrieve("project alpha", top_k=10, source="obsidian")
+ for r in obsidian_results:
+ assert r.metadata.get("source") == "obsidian"
+
+ assert len(gmail_results) >= 1
+ assert len(obsidian_results) >= 1
+
+
+def test_retrieve_filter_by_doc_type(ks: KnowledgeStore) -> None:
+ """retrieve() with doc_type= filters correctly."""
+ _store(ks, content="Meeting notes from Monday", source="calendar", doc_type="meeting")
+ _store(ks, content="Meeting email received Tuesday", source="gmail", doc_type="email")
+
+ meeting_results = ks.retrieve("meeting", top_k=10, doc_type="meeting")
+ for r in meeting_results:
+ assert r.metadata.get("doc_type") == "meeting"
+
+ email_results = ks.retrieve("meeting", top_k=10, doc_type="email")
+ for r in email_results:
+ assert r.metadata.get("doc_type") == "email"
+
+ assert len(meeting_results) >= 1
+ assert len(email_results) >= 1
+
+
+def test_retrieve_filter_by_author(ks: KnowledgeStore) -> None:
+ """retrieve() with author= filters correctly."""
+ _store(ks, content="Alice wrote about machine learning research", source="gmail", doc_type="email", author="[email protected]")
+ _store(ks, content="Bob wrote about machine learning deployment", source="gmail", doc_type="email", author="[email protected]")
+
+ alice_results = ks.retrieve("machine learning", top_k=10, author="[email protected]")
+ for r in alice_results:
+ assert r.metadata.get("author") == "[email protected]"
+
+ bob_results = ks.retrieve("machine learning", top_k=10, author="[email protected]")
+ for r in bob_results:
+ assert r.metadata.get("author") == "[email protected]"
+
+ assert len(alice_results) >= 1
+ assert len(bob_results) >= 1
+
+
+def test_retrieve_filter_by_timestamp_since(ks: KnowledgeStore) -> None:
+ """retrieve() with since= excludes older documents."""
+ old_ts = datetime(2023, 1, 1, tzinfo=timezone.utc)
+ new_ts = datetime(2025, 6, 1, tzinfo=timezone.utc)
+ cutoff = datetime(2024, 1, 1, tzinfo=timezone.utc)
+
+ _store(ks, content="Old document about research", source="notes", timestamp=old_ts)
+ _store(ks, content="New document about research", source="notes", timestamp=new_ts)
+
+ results = ks.retrieve("document research", top_k=10, since=cutoff)
+ for r in results:
+ ts_str = r.metadata.get("timestamp", "")
+ assert ts_str >= cutoff.isoformat(), f"Got old doc in results: {ts_str}"
+
+ assert len(results) >= 1
+
+
+def test_delete_by_doc_id(ks: KnowledgeStore) -> None:
+ """delete() removes all chunks with matching doc_id."""
+ doc_id = "test:doc:001"
+ _store(ks, content="First chunk of the document", source="notes", doc_id=doc_id, chunk_index=0)
+ _store(ks, content="Second chunk of the document", source="notes", doc_id=doc_id, chunk_index=1)
+ _store(ks, content="Other document content", source="notes", doc_id="other:doc:001", chunk_index=0)
+
+ # Confirm items are stored
+ before = ks.retrieve("document", top_k=10)
+ assert len(before) >= 3
+
+ deleted = ks.delete(doc_id)
+ assert deleted is True
+
+ # Only the other doc should remain
+ after = ks.retrieve("document", top_k=10)
+ for r in after:
+ assert r.metadata.get("doc_id") != doc_id
+
+ # Deleting non-existent doc_id returns False
+ assert ks.delete("nonexistent:doc") is False
+
+
+def test_clear(ks: KnowledgeStore) -> None:
+ """clear() removes all stored documents."""
+ _store(ks, content="Document one about research")
+ _store(ks, content="Document two about analysis")
+ _store(ks, content="Document three about synthesis")
+
+ before = ks.retrieve("document", top_k=10)
+ assert len(before) >= 3
+
+ ks.clear()
+
+ after = ks.retrieve("document", top_k=10)
+ assert len(after) == 0
+
+
+def test_store_with_metadata(ks: KnowledgeStore) -> None:
+ """Extra metadata and thread_id are preserved in retrieval results."""
+ custom_meta = {"labels": ["important", "action-needed"], "priority": "high"}
+ chunk_id = _store(
+ ks,
+ content="Meeting follow-up with action items",
+ source="gmail",
+ doc_type="email",
+ thread_id="thread:abc:123",
+ metadata=custom_meta,
+ )
+ assert chunk_id # non-empty string
+
+ results = ks.retrieve("action items", top_k=5)
+ assert len(results) >= 1
+ meta = results[0].metadata
+ assert meta.get("thread_id") == "thread:abc:123"
+ # Custom metadata should also be available
+ assert "labels" in meta or meta.get("metadata", {}).get("labels") is not None
+
+
+def test_store_preserves_url(ks: KnowledgeStore) -> None:
+ """URL field survives the store/retrieve round-trip."""
+ url = "https://mail.google.com/mail/u/0/#inbox/abc123"
+ _store(
+ ks,
+ content="Email with a specific URL for testing",
+ source="gmail",
+ doc_type="email",
+ url=url,
+ )
+ results = ks.retrieve("specific URL testing", top_k=5)
+ assert len(results) >= 1
+ assert results[0].metadata.get("url") == url
+
+
+def test_retrieve_empty_store(ks: KnowledgeStore) -> None:
+ """Querying an empty store returns an empty list (no crash)."""
+ results = ks.retrieve("anything", top_k=5)
+ assert results == []
+
+
+def test_retrieve_top_k_respected(ks: KnowledgeStore) -> None:
+ """top_k limits the number of returned results."""
+ for i in range(10):
+ _store(ks, content=f"Research document number {i} about topic X", source="notes")
+
+ results = ks.retrieve("research topic", top_k=3)
+ assert len(results) <= 3
+
+
+def test_store_returns_unique_chunk_ids(ks: KnowledgeStore) -> None:
+ """Each store() call returns a unique chunk id."""
+ id1 = _store(ks, content="First chunk")
+ id2 = _store(ks, content="Second chunk")
+ assert id1 != id2
+
+
+def test_retrieve_filter_by_until(ks: KnowledgeStore) -> None:
+ """retrieve() with until= excludes newer documents."""
+ old_ts = datetime(2023, 6, 1, tzinfo=timezone.utc)
+ new_ts = datetime(2025, 6, 1, tzinfo=timezone.utc)
+ cutoff = datetime(2024, 1, 1, tzinfo=timezone.utc)
+
+ _store(ks, content="Old quarterly report analysis", source="notes", timestamp=old_ts)
+ _store(ks, content="New quarterly report analysis", source="notes", timestamp=new_ts)
+
+ results = ks.retrieve("quarterly report", top_k=10, until=cutoff)
+ for r in results:
+ ts_str = r.metadata.get("timestamp", "")
+ assert ts_str <= cutoff.isoformat(), f"Got new doc in results: {ts_str}"
+
+ assert len(results) >= 1
+
+
+def test_memory_store_event_emitted(tmp_path: Path) -> None:
+ """MEMORY_STORE event is published on store()."""
+ from openjarvis.core.events import EventType, get_event_bus, reset_event_bus
+
+ reset_event_bus()
+ bus = get_event_bus(record_history=True)
+ ks = KnowledgeStore(db_path=tmp_path / "events.db")
+
+ _store(ks, content="Event emission test content", source="test")
+
+ types = [e.event_type for e in bus.history]
+ assert EventType.MEMORY_STORE in types
+
+
+def test_memory_retrieve_event_emitted(tmp_path: Path) -> None:
+ """MEMORY_RETRIEVE event is published on retrieve()."""
+ from openjarvis.core.events import EventType, get_event_bus, reset_event_bus
+
+ reset_event_bus()
+ bus = get_event_bus(record_history=True)
+ ks = KnowledgeStore(db_path=tmp_path / "events2.db")
+
+ _store(ks, content="Some content to retrieve", source="test")
+ ks.retrieve("some content", top_k=5)
+
+ types = [e.event_type for e in bus.history]
+ assert EventType.MEMORY_RETRIEVE in types