Personal AI, On Personal Devices
Python
feat: add knowledge_search tool with filtered BM25 retrieval and source attribution
krypticmouseffc43cd7b5cb
diff --git a/src/openjarvis/tools/knowledge_search.py b/src/openjarvis/tools/knowledge_search.pynew file mode 100644index 00000000..803d8b05--- /dev/null+++ b/src/openjarvis/tools/knowledge_search.py@@ -0,0 +1,160 @@+"""KnowledgeSearchTool — filtered BM25 retrieval with source attribution.++Wraps ``KnowledgeStore`` so agents can search ingested documents by text query+and optional provenance filters (source, doc_type, author, date range).+"""++from __future__ import annotations++from typing import Any, Optional++from openjarvis.connectors.store import KnowledgeStore+from openjarvis.core.registry import ToolRegistry+from openjarvis.core.types import ToolResult+from openjarvis.tools._stubs import BaseTool, ToolSpec++[email protected]("knowledge_search")+class KnowledgeSearchTool(BaseTool):+ """Search the knowledge store using filtered BM25 retrieval.++ Results include source attribution so agents can cite provenance.+ """++ tool_id = "knowledge_search"++ def __init__(self, store: Optional[KnowledgeStore] = None) -> None:+ self._store = store++ @property+ def spec(self) -> ToolSpec:+ return ToolSpec(+ name="knowledge_search",+ description=(+ "Search ingested personal knowledge (emails, Slack messages,"+ " documents) using full-text BM25 retrieval with optional"+ " filters for source, type, author, and date range."+ ),+ parameters={+ "type": "object",+ "properties": {+ "query": {+ "type": "string",+ "description": "Full-text search query.",+ },+ "source": {+ "type": "string",+ "description": (+ "Filter by source connector"+ " (e.g. 'gmail', 'slack', 'obsidian')."+ ),+ },+ "doc_type": {+ "type": "string",+ "description": (+ "Filter by document type"+ " (e.g. 'email', 'message', 'document')."+ ),+ },+ "author": {+ "type": "string",+ "description": "Filter by author.",+ },+ "since": {+ "type": "string",+ "description": (+ "Exclude documents before this ISO 8601 timestamp."+ ),+ },+ "until": {+ "type": "string",+ "description": (+ "Exclude documents after this ISO 8601 timestamp."+ ),+ },+ "top_k": {+ "type": "integer",+ "description": "Maximum number of results (default 10).",+ },+ },+ "required": ["query"],+ },+ category="knowledge",+ )++ def execute(self, **params: Any) -> ToolResult:+ if self._store is None:+ return ToolResult(+ tool_name="knowledge_search",+ content="No knowledge store configured.",+ success=False,+ )++ query: str = params.get("query", "")+ if not query:+ return ToolResult(+ tool_name="knowledge_search",+ content="No query provided.",+ success=False,+ )++ top_k: int = int(params.get("top_k", 10))+ source: Optional[str] = params.get("source")+ doc_type: Optional[str] = params.get("doc_type")+ author: Optional[str] = params.get("author")+ since: Optional[str] = params.get("since")+ until: Optional[str] = params.get("until")++ results = self._store.retrieve(+ query,+ top_k=top_k,+ source=source,+ doc_type=doc_type,+ author=author,+ since=since,+ until=until,+ )++ if not results:+ return ToolResult(+ tool_name="knowledge_search",+ content="No relevant results found.",+ success=True,+ metadata={"num_results": 0},+ )++ lines: list[str] = []+ for i, result in enumerate(results, start=1):+ meta = result.metadata+ src_label = result.source or meta.get("source", "")+ title = meta.get("title", "")+ result_author = meta.get("author", "")+ url = meta.get("url", "")++ # Build header line+ header_parts: list[str] = []+ if src_label:+ header_parts.append(f"[{src_label}]")+ if title:+ header_parts.append(title)+ if result_author:+ header_parts.append(f"by {result_author}")+ if url:+ header_parts.append(f"({url})")++ header = " ".join(header_parts) if header_parts else "(unknown source)"+ lines.append(f"**Result {i}:** {header}")+ lines.append(result.content)+ lines.append("")++ formatted = "\n".join(lines).rstrip()++ return ToolResult(+ tool_name="knowledge_search",+ content=formatted,+ success=True,+ metadata={"num_results": len(results)},+ )+++__all__ = ["KnowledgeSearchTool"]diff --git a/tests/tools/test_knowledge_search.py b/tests/tools/test_knowledge_search.pynew file mode 100644index 00000000..e6b32e50--- /dev/null+++ b/tests/tools/test_knowledge_search.py@@ -0,0 +1,139 @@+"""Tests for the knowledge_search tool."""++from __future__ import annotations++import importlib+import sys++import pytest++from openjarvis.connectors.store import KnowledgeStore+from openjarvis.core.registry import ToolRegistry+from openjarvis.tools.knowledge_search import KnowledgeSearchTool++# ---------------------------------------------------------------------------+# Fixture+# ---------------------------------------------------------------------------+++def store(tmp_path):+ """Return a KnowledgeStore pre-loaded with 3 diverse items."""+ s = KnowledgeStore(db_path=tmp_path / "test_knowledge.db")++ # Item 1 — gmail email from alice+ s.store(+ "Meeting about Kubernetes migration scheduled for next Tuesday.",+ source="gmail",+ doc_type="email",+ title="Re: K8s migration",+ author="[email protected]",+ url="https://mail.google.com/mail/u/0/#inbox/abc123",+ timestamp="2026-01-15T10:00:00Z",+ )++ # Item 2 — slack message from bob+ s.store(+ "Discussion about K8s costs — we should consider spot instances.",+ source="slack",+ doc_type="message",+ title="#infrastructure",+ author="[email protected]",+ url="slack://thread/def456",+ timestamp="2026-01-20T14:30:00Z",+ )++ # Item 3 — obsidian document from sarah+ s.store(+ "Research notes on large language model fine-tuning strategies.",+ source="obsidian",+ doc_type="document",+ title="LLM Fine-tuning Notes",+ author="[email protected]",+ url="obsidian://vault/llm-notes",+ timestamp="2026-02-01T09:00:00Z",+ )++ yield s+ s.close()+++# ---------------------------------------------------------------------------+# Tests+# ---------------------------------------------------------------------------+++class TestKnowledgeSearchTool:+ def test_basic_search(self, store):+ """Search finds the matching document from the 3-item store."""+ tool = KnowledgeSearchTool(store=store)+ result = tool.execute(query="Kubernetes migration")+ assert result.success is True+ assert "Kubernetes" in result.content+ assert result.metadata["num_results"] >= 1++ def test_filter_by_source(self, store):+ """source filter restricts results to gmail only."""+ tool = KnowledgeSearchTool(store=store)+ result = tool.execute(query="Kubernetes", source="gmail")+ assert result.success is True+ assert "gmail" in result.content+ # Every returned result must come from gmail+ assert "slack" not in result.content++ def test_filter_by_author(self, store):+ """author filter restricts results to sarah only."""+ tool = KnowledgeSearchTool(store=store)+ # Use "language model" — FTS5 does not tokenise hyphenated terms like+ # "fine-tuning" as a single token, so we search for a phrase that works.+ result = tool.execute(query="language model", author="[email protected]")+ assert result.success is True+ assert "[email protected]" in result.content+ assert result.metadata["num_results"] >= 1++ def test_no_results(self, store):+ """Query matching nothing returns success=True with 'No relevant results'."""+ tool = KnowledgeSearchTool(store=store)+ result = tool.execute(query="zzz_nonexistent_xyzzy_12345")+ assert result.success is True+ assert "No relevant results" in result.content+ assert result.metadata["num_results"] == 0++ def test_empty_query(self, store):+ """Empty query string returns success=False."""+ tool = KnowledgeSearchTool(store=store)+ result = tool.execute(query="")+ assert result.success is False+ assert "No query provided" in result.content++ def test_no_store(self):+ """Missing store returns success=False."""+ tool = KnowledgeSearchTool()+ result = tool.execute(query="kubernetes")+ assert result.success is False+ assert "No knowledge store configured" in result.content++ def test_spec_has_filter_params(self):+ """ToolSpec.parameters includes all required and optional filter fields."""+ tool = KnowledgeSearchTool()+ props = tool.spec.parameters.get("properties", {})+ for field in ("query", "source", "doc_type", "author", "since", "top_k"):+ assert field in props, f"Missing parameter: {field}"+ assert "query" in tool.spec.parameters.get("required", [])+ assert tool.spec.category == "knowledge"++ def test_registry(self):+ """ToolRegistry contains 'knowledge_search' after module import.++ The autouse ``_clean_registries`` fixture clears all registries before+ each test. Since the module is already cached in ``sys.modules`` a+ plain import won't re-execute the ``@ToolRegistry.register`` decorator,+ so we explicitly reload the module.+ """+ mod_name = "openjarvis.tools.knowledge_search"+ if mod_name in sys.modules:+ importlib.reload(sys.modules[mod_name])+ else:+ importlib.import_module(mod_name)++ assert ToolRegistry.contains("knowledge_search")