Personal AI, On Personal Devices
Python
feat: add KnowledgeStore with source-aware SQLite schema and filtered BM25 retrieval
krypticmouse8d3505f8f34e
diff --git a/src/openjarvis/connectors/__init__.py b/src/openjarvis/connectors/__init__.pyindex 1642e5bd..a48d0d71 100644--- a/src/openjarvis/connectors/__init__.py+++ b/src/openjarvis/connectors/__init__.py@@ -6,5 +6,6 @@ from openjarvis.connectors._stubs import (Document,SyncStatus,)+from openjarvis.connectors.store import KnowledgeStore-__all__ = ["Attachment", "BaseConnector", "Document", "SyncStatus"]+__all__ = ["Attachment", "BaseConnector", "Document", "KnowledgeStore", "SyncStatus"]diff --git a/src/openjarvis/connectors/store.py b/src/openjarvis/connectors/store.pynew file mode 100644index 00000000..9e65ff2d--- /dev/null+++ b/src/openjarvis/connectors/store.py@@ -0,0 +1,371 @@+"""KnowledgeStore — source-aware SQLite/FTS5 memory backend for Deep Research.++Extends ``MemoryBackend`` with per-document provenance columns so that the+IngestionPipeline and the ``knowledge_search`` tool can filter results by+source, doc_type, author, and timestamp ranges.++Pure Python ``sqlite3`` (no Rust extension required).+"""++from __future__ import annotations++import json+import sqlite3+import time+import uuid+from datetime import datetime, timezone+from pathlib import Path+from typing import Any, Dict, List, Optional, Union++from openjarvis.core.events import EventType, get_event_bus+from openjarvis.core.registry import MemoryRegistry+from openjarvis.tools.storage._stubs import MemoryBackend, RetrievalResult++# ---------------------------------------------------------------------------+# DDL+# ---------------------------------------------------------------------------++_CREATE_MAIN_TABLE = """+CREATE TABLE IF NOT EXISTS knowledge_chunks (+ id TEXT PRIMARY KEY,+ doc_id TEXT NOT NULL,+ content TEXT NOT NULL,+ source TEXT NOT NULL DEFAULT '',+ doc_type TEXT NOT NULL DEFAULT '',+ title TEXT NOT NULL DEFAULT '',+ author TEXT NOT NULL DEFAULT '',+ participants TEXT NOT NULL DEFAULT '[]',+ timestamp TEXT NOT NULL DEFAULT '',+ thread_id TEXT NOT NULL DEFAULT '',+ url TEXT NOT NULL DEFAULT '',+ metadata TEXT NOT NULL DEFAULT '{}',+ chunk_index INTEGER NOT NULL DEFAULT 0,+ created_at REAL NOT NULL+);+"""++_CREATE_FTS_TABLE = """+CREATE VIRTUAL TABLE IF NOT EXISTS knowledge_fts+USING fts5(+ content,+ title,+ author,+ content='knowledge_chunks',+ content_rowid='rowid',+ tokenize='porter unicode61'+);+"""++_CREATE_TRIGGERS = """+CREATE TRIGGER IF NOT EXISTS knowledge_ai AFTER INSERT ON knowledge_chunks BEGIN+ INSERT INTO knowledge_fts(rowid, content, title, author)+ VALUES (new.rowid, new.content, new.title, new.author);+END;++CREATE TRIGGER IF NOT EXISTS knowledge_ad AFTER DELETE ON knowledge_chunks BEGIN+ INSERT INTO knowledge_fts(knowledge_fts, rowid, content, title, author)+ VALUES ('delete', old.rowid, old.content, old.title, old.author);+END;++CREATE TRIGGER IF NOT EXISTS knowledge_au AFTER UPDATE ON knowledge_chunks BEGIN+ INSERT INTO knowledge_fts(knowledge_fts, rowid, content, title, author)+ VALUES ('delete', old.rowid, old.content, old.title, old.author);+ INSERT INTO knowledge_fts(rowid, content, title, author)+ VALUES (new.rowid, new.content, new.title, new.author);+END;+"""++_CREATE_INDEXES = """+CREATE INDEX IF NOT EXISTS idx_kc_source ON knowledge_chunks(source);+CREATE INDEX IF NOT EXISTS idx_kc_doc_type ON knowledge_chunks(doc_type);+CREATE INDEX IF NOT EXISTS idx_kc_author ON knowledge_chunks(author);+CREATE INDEX IF NOT EXISTS idx_kc_timestamp ON knowledge_chunks(timestamp);+CREATE INDEX IF NOT EXISTS idx_kc_thread_id ON knowledge_chunks(thread_id);+CREATE INDEX IF NOT EXISTS idx_kc_doc_id ON knowledge_chunks(doc_id);+"""++# ---------------------------------------------------------------------------+# Helper+# ---------------------------------------------------------------------------+++def _to_iso(ts: Optional[Union[datetime, str]]) -> str:+ """Normalise a timestamp to ISO 8601 string (UTC)."""+ if ts is None:+ return ""+ if isinstance(ts, str):+ return ts+ if ts.tzinfo is None:+ ts = ts.replace(tzinfo=timezone.utc)+ return ts.isoformat()+++# ---------------------------------------------------------------------------+# KnowledgeStore+# ---------------------------------------------------------------------------++[email protected]("knowledge")+class KnowledgeStore(MemoryBackend):+ """Source-aware SQLite/FTS5 knowledge store for Deep Research.++ Stores document chunks with rich provenance metadata and supports+ filtered BM25 retrieval by source, doc_type, author, and timestamp.+ """++ backend_id: str = "knowledge"++ def __init__(self, db_path: Union[str, Path] = "") -> None:+ if not db_path:+ from openjarvis.core.config import DEFAULT_CONFIG_DIR+ db_path = DEFAULT_CONFIG_DIR / "knowledge.db"++ self._db_path = str(db_path)+ # Ensure the parent directory exists (skip for :memory:)+ if self._db_path != ":memory:":+ Path(self._db_path).parent.mkdir(parents=True, exist_ok=True)++ self._conn = sqlite3.connect(self._db_path, check_same_thread=False)+ self._conn.row_factory = sqlite3.Row+ self._setup()++ # ------------------------------------------------------------------+ # Internal setup+ # ------------------------------------------------------------------++ def _setup(self) -> None:+ """Create tables, FTS virtual table, triggers and indexes."""+ self._conn.execute("PRAGMA journal_mode=WAL;")+ self._conn.execute("PRAGMA foreign_keys=ON;")+ self._conn.executescript(+ _CREATE_MAIN_TABLE+ + _CREATE_FTS_TABLE+ + _CREATE_TRIGGERS+ + _CREATE_INDEXES+ )+ self._conn.commit()++ # ------------------------------------------------------------------+ # MemoryBackend interface+ # ------------------------------------------------------------------++ def store( # type: ignore[override]+ self,+ content: str,+ *,+ source: str = "",+ doc_type: str = "",+ doc_id: Optional[str] = None,+ title: str = "",+ author: str = "",+ participants: Optional[List[str]] = None,+ timestamp: Optional[Union[datetime, str]] = None,+ thread_id: Optional[str] = None,+ url: Optional[str] = None,+ metadata: Optional[Dict[str, Any]] = None,+ chunk_index: int = 0,+ ) -> str:+ """Persist a content chunk and return its unique chunk id.++ All source-level fields are merged into the stored metadata so that+ ``retrieve()`` results carry full provenance.+ """+ chunk_id = str(uuid.uuid4())+ if doc_id is None:+ doc_id = str(uuid.uuid4())++ ts_str = _to_iso(timestamp)+ participants_json = json.dumps(participants or [])++ # Merge provenance fields into metadata for easy access in results+ combined_meta: Dict[str, Any] = dict(metadata or {})+ combined_meta["source"] = source+ combined_meta["doc_type"] = doc_type+ combined_meta["doc_id"] = doc_id+ combined_meta["title"] = title+ combined_meta["author"] = author+ combined_meta["participants"] = participants or []+ combined_meta["timestamp"] = ts_str+ combined_meta["thread_id"] = thread_id or ""+ combined_meta["url"] = url or ""+ combined_meta["chunk_index"] = chunk_index++ meta_json = json.dumps(combined_meta)++ self._conn.execute(+ """+ INSERT INTO knowledge_chunks+ (id, doc_id, content, source, doc_type, title, author,+ participants, timestamp, thread_id, url, metadata,+ chunk_index, created_at)+ VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)+ """,+ (+ chunk_id,+ doc_id,+ content,+ source,+ doc_type,+ title,+ author,+ participants_json,+ ts_str,+ thread_id or "",+ url or "",+ meta_json,+ chunk_index,+ time.time(),+ ),+ )+ self._conn.commit()++ get_event_bus().publish(+ EventType.MEMORY_STORE,+ {+ "backend": self.backend_id,+ "chunk_id": chunk_id,+ "doc_id": doc_id,+ "source": source,+ "doc_type": doc_type,+ },+ )+ return chunk_id++ def retrieve(+ self,+ query: str,+ *,+ top_k: int = 5,+ source: Optional[str] = None,+ doc_type: Optional[str] = None,+ author: Optional[str] = None,+ since: Optional[Union[datetime, str]] = None,+ until: Optional[Union[datetime, str]] = None,+ **kwargs: Any,+ ) -> List[RetrievalResult]:+ """Search using FTS5 BM25 with optional column filters.++ Parameters+ ----------+ query: Full-text search query.+ top_k: Maximum number of results.+ source: Restrict to chunks from this source (e.g. "gmail").+ doc_type: Restrict to chunks of this type (e.g. "email").+ author: Restrict to chunks authored by this person.+ since: Exclude chunks whose timestamp is earlier than this value.+ until: Exclude chunks whose timestamp is later than this value.+ """+ if not query.strip():+ return []++ since_str = _to_iso(since) if since is not None else None+ until_str = _to_iso(until) if until is not None else None++ # Build the WHERE clause for filter columns+ filters: List[str] = []+ params: List[Any] = []++ if source is not None:+ filters.append("kc.source = ?")+ params.append(source)+ if doc_type is not None:+ filters.append("kc.doc_type = ?")+ params.append(doc_type)+ if author is not None:+ filters.append("kc.author = ?")+ params.append(author)+ if since_str:+ filters.append("kc.timestamp >= ?")+ params.append(since_str)+ if until_str:+ filters.append("kc.timestamp <= ?")+ params.append(until_str)++ where_clause = ""+ if filters:+ where_clause = "AND " + " AND ".join(filters)++ # FTS5 bm25() returns negative scores; abs() gives a positive rank+ sql = f"""+ SELECT+ kc.id,+ kc.content,+ kc.source,+ kc.metadata,+ abs(bm25(knowledge_fts)) AS score+ FROM knowledge_fts+ JOIN knowledge_chunks kc ON knowledge_fts.rowid = kc.rowid+ WHERE knowledge_fts MATCH ?+ {where_clause}+ ORDER BY score DESC+ LIMIT ?+ """++ try:+ rows = self._conn.execute(sql, [query] + params + [top_k]).fetchall()+ except sqlite3.OperationalError:+ # Malformed FTS query — return empty rather than crash+ return []++ results: List[RetrievalResult] = []+ for row in rows:+ meta = json.loads(row["metadata"]) if row["metadata"] else {}+ results.append(+ RetrievalResult(+ content=row["content"],+ score=float(row["score"]),+ source=row["source"],+ metadata=meta,+ )+ )++ get_event_bus().publish(+ EventType.MEMORY_RETRIEVE,+ {+ "backend": self.backend_id,+ "query": query,+ "num_results": len(results),+ "filters": {+ "source": source,+ "doc_type": doc_type,+ "author": author,+ "since": since_str,+ "until": until_str,+ },+ },+ )+ return results++ def delete(self, doc_id: str) -> bool:+ """Delete all chunks with the given *doc_id*. Returns True if any existed."""+ cur = self._conn.execute(+ "DELETE FROM knowledge_chunks WHERE doc_id = ?", (doc_id,)+ )+ self._conn.commit()+ return cur.rowcount > 0++ def clear(self) -> None:+ """Remove all stored chunks."""+ self._conn.executescript(+ "DELETE FROM knowledge_chunks; DELETE FROM knowledge_fts;"+ )+ self._conn.commit()++ # ------------------------------------------------------------------+ # Extra helpers+ # ------------------------------------------------------------------++ def count(self) -> int:+ """Return the total number of stored chunks."""+ row = self._conn.execute("SELECT COUNT(*) FROM knowledge_chunks").fetchone()+ return row[0] if row else 0++ def close(self) -> None:+ """Close the underlying SQLite connection."""+ try:+ self._conn.close()+ except Exception:+ pass+++__all__ = ["KnowledgeStore"]diff --git a/tests/connectors/test_store.py b/tests/connectors/test_store.pynew file mode 100644index 00000000..90700dfe--- /dev/null+++ b/tests/connectors/test_store.py@@ -0,0 +1,271 @@+"""Tests for KnowledgeStore — source-aware SQLite/FTS5 memory backend."""++from __future__ import annotations++import time+from datetime import datetime, timezone+from pathlib import Path++import pytest++from openjarvis.connectors.store import KnowledgeStore+++# ---------------------------------------------------------------------------+# Helpers+# ---------------------------------------------------------------------------+++def _store(ks: KnowledgeStore, **kwargs) -> str: # type: ignore[type-arg]+ """Convenience wrapper with sensible defaults."""+ defaults = {+ "content": "default content",+ "source": "test",+ "doc_type": "note",+ "doc_id": None,+ "title": "",+ "author": "",+ "participants": None,+ "timestamp": None,+ "thread_id": None,+ "url": None,+ "metadata": None,+ "chunk_index": 0,+ }+ defaults.update(kwargs)+ return ks.store(**defaults) # type: ignore[call-arg]+++# ---------------------------------------------------------------------------+# Fixtures+# ---------------------------------------------------------------------------+++def ks(tmp_path: Path) -> KnowledgeStore:+ db = tmp_path / "test_knowledge.db"+ return KnowledgeStore(db_path=db)+++# ---------------------------------------------------------------------------+# Tests+# ---------------------------------------------------------------------------+++def test_store_and_retrieve_basic(ks: KnowledgeStore) -> None:+ """Stored content can be retrieved by a matching query."""+ _store(ks, content="The quick brown fox jumps over the lazy dog", source="notes", doc_type="note")+ results = ks.retrieve("quick brown fox", top_k=5)+ assert len(results) >= 1+ assert "fox" in results[0].content.lower() or "quick" in results[0].content.lower()+ assert results[0].score > 0.0+++def test_retrieve_filter_by_source(ks: KnowledgeStore) -> None:+ """retrieve() with source= returns only chunks from that source."""+ _store(ks, content="Email about project alpha", source="gmail", doc_type="email")+ _store(ks, content="Note about project alpha progress", source="obsidian", doc_type="note")++ gmail_results = ks.retrieve("project alpha", top_k=10, source="gmail")+ for r in gmail_results:+ assert r.metadata.get("source") == "gmail"++ obsidian_results = ks.retrieve("project alpha", top_k=10, source="obsidian")+ for r in obsidian_results:+ assert r.metadata.get("source") == "obsidian"++ assert len(gmail_results) >= 1+ assert len(obsidian_results) >= 1+++def test_retrieve_filter_by_doc_type(ks: KnowledgeStore) -> None:+ """retrieve() with doc_type= filters correctly."""+ _store(ks, content="Meeting notes from Monday", source="calendar", doc_type="meeting")+ _store(ks, content="Meeting email received Tuesday", source="gmail", doc_type="email")++ meeting_results = ks.retrieve("meeting", top_k=10, doc_type="meeting")+ for r in meeting_results:+ assert r.metadata.get("doc_type") == "meeting"++ email_results = ks.retrieve("meeting", top_k=10, doc_type="email")+ for r in email_results:+ assert r.metadata.get("doc_type") == "email"++ assert len(meeting_results) >= 1+ assert len(email_results) >= 1+++def test_retrieve_filter_by_author(ks: KnowledgeStore) -> None:+ """retrieve() with author= filters correctly."""+ _store(ks, content="Alice wrote about machine learning research", source="gmail", doc_type="email", author="[email protected]")+ _store(ks, content="Bob wrote about machine learning deployment", source="gmail", doc_type="email", author="[email protected]")++ alice_results = ks.retrieve("machine learning", top_k=10, author="[email protected]")+ for r in alice_results:+ assert r.metadata.get("author") == "[email protected]"++ bob_results = ks.retrieve("machine learning", top_k=10, author="[email protected]")+ for r in bob_results:+ assert r.metadata.get("author") == "[email protected]"++ assert len(alice_results) >= 1+ assert len(bob_results) >= 1+++def test_retrieve_filter_by_timestamp_since(ks: KnowledgeStore) -> None:+ """retrieve() with since= excludes older documents."""+ old_ts = datetime(2023, 1, 1, tzinfo=timezone.utc)+ new_ts = datetime(2025, 6, 1, tzinfo=timezone.utc)+ cutoff = datetime(2024, 1, 1, tzinfo=timezone.utc)++ _store(ks, content="Old document about research", source="notes", timestamp=old_ts)+ _store(ks, content="New document about research", source="notes", timestamp=new_ts)++ results = ks.retrieve("document research", top_k=10, since=cutoff)+ for r in results:+ ts_str = r.metadata.get("timestamp", "")+ assert ts_str >= cutoff.isoformat(), f"Got old doc in results: {ts_str}"++ assert len(results) >= 1+++def test_delete_by_doc_id(ks: KnowledgeStore) -> None:+ """delete() removes all chunks with matching doc_id."""+ doc_id = "test:doc:001"+ _store(ks, content="First chunk of the document", source="notes", doc_id=doc_id, chunk_index=0)+ _store(ks, content="Second chunk of the document", source="notes", doc_id=doc_id, chunk_index=1)+ _store(ks, content="Other document content", source="notes", doc_id="other:doc:001", chunk_index=0)++ # Confirm items are stored+ before = ks.retrieve("document", top_k=10)+ assert len(before) >= 3++ deleted = ks.delete(doc_id)+ assert deleted is True++ # Only the other doc should remain+ after = ks.retrieve("document", top_k=10)+ for r in after:+ assert r.metadata.get("doc_id") != doc_id++ # Deleting non-existent doc_id returns False+ assert ks.delete("nonexistent:doc") is False+++def test_clear(ks: KnowledgeStore) -> None:+ """clear() removes all stored documents."""+ _store(ks, content="Document one about research")+ _store(ks, content="Document two about analysis")+ _store(ks, content="Document three about synthesis")++ before = ks.retrieve("document", top_k=10)+ assert len(before) >= 3++ ks.clear()++ after = ks.retrieve("document", top_k=10)+ assert len(after) == 0+++def test_store_with_metadata(ks: KnowledgeStore) -> None:+ """Extra metadata and thread_id are preserved in retrieval results."""+ custom_meta = {"labels": ["important", "action-needed"], "priority": "high"}+ chunk_id = _store(+ ks,+ content="Meeting follow-up with action items",+ source="gmail",+ doc_type="email",+ thread_id="thread:abc:123",+ metadata=custom_meta,+ )+ assert chunk_id # non-empty string++ results = ks.retrieve("action items", top_k=5)+ assert len(results) >= 1+ meta = results[0].metadata+ assert meta.get("thread_id") == "thread:abc:123"+ # Custom metadata should also be available+ assert "labels" in meta or meta.get("metadata", {}).get("labels") is not None+++def test_store_preserves_url(ks: KnowledgeStore) -> None:+ """URL field survives the store/retrieve round-trip."""+ url = "https://mail.google.com/mail/u/0/#inbox/abc123"+ _store(+ ks,+ content="Email with a specific URL for testing",+ source="gmail",+ doc_type="email",+ url=url,+ )+ results = ks.retrieve("specific URL testing", top_k=5)+ assert len(results) >= 1+ assert results[0].metadata.get("url") == url+++def test_retrieve_empty_store(ks: KnowledgeStore) -> None:+ """Querying an empty store returns an empty list (no crash)."""+ results = ks.retrieve("anything", top_k=5)+ assert results == []+++def test_retrieve_top_k_respected(ks: KnowledgeStore) -> None:+ """top_k limits the number of returned results."""+ for i in range(10):+ _store(ks, content=f"Research document number {i} about topic X", source="notes")++ results = ks.retrieve("research topic", top_k=3)+ assert len(results) <= 3+++def test_store_returns_unique_chunk_ids(ks: KnowledgeStore) -> None:+ """Each store() call returns a unique chunk id."""+ id1 = _store(ks, content="First chunk")+ id2 = _store(ks, content="Second chunk")+ assert id1 != id2+++def test_retrieve_filter_by_until(ks: KnowledgeStore) -> None:+ """retrieve() with until= excludes newer documents."""+ old_ts = datetime(2023, 6, 1, tzinfo=timezone.utc)+ new_ts = datetime(2025, 6, 1, tzinfo=timezone.utc)+ cutoff = datetime(2024, 1, 1, tzinfo=timezone.utc)++ _store(ks, content="Old quarterly report analysis", source="notes", timestamp=old_ts)+ _store(ks, content="New quarterly report analysis", source="notes", timestamp=new_ts)++ results = ks.retrieve("quarterly report", top_k=10, until=cutoff)+ for r in results:+ ts_str = r.metadata.get("timestamp", "")+ assert ts_str <= cutoff.isoformat(), f"Got new doc in results: {ts_str}"++ assert len(results) >= 1+++def test_memory_store_event_emitted(tmp_path: Path) -> None:+ """MEMORY_STORE event is published on store()."""+ from openjarvis.core.events import EventType, get_event_bus, reset_event_bus++ reset_event_bus()+ bus = get_event_bus(record_history=True)+ ks = KnowledgeStore(db_path=tmp_path / "events.db")++ _store(ks, content="Event emission test content", source="test")++ types = [e.event_type for e in bus.history]+ assert EventType.MEMORY_STORE in types+++def test_memory_retrieve_event_emitted(tmp_path: Path) -> None:+ """MEMORY_RETRIEVE event is published on retrieve()."""+ from openjarvis.core.events import EventType, get_event_bus, reset_event_bus++ reset_event_bus()+ bus = get_event_bus(record_history=True)+ ks = KnowledgeStore(db_path=tmp_path / "events2.db")++ _store(ks, content="Some content to retrieve", source="test")+ ks.retrieve("some content", top_k=5)++ types = [e.event_type for e in bus.history]+ assert EventType.MEMORY_RETRIEVE in types