Skip to content

Personal AI, On Personal Devices

Python

feat: add knowledge_search tool with filtered BM25 retrieval and source attribution

krypticmouseffc43cd7b5cb
diff --git a/src/openjarvis/tools/knowledge_search.py b/src/openjarvis/tools/knowledge_search.py
new file mode 100644
index 00000000..803d8b05
--- /dev/null
+++ b/src/openjarvis/tools/knowledge_search.py
@@ -0,0 +1,160 @@
+"""KnowledgeSearchTool — filtered BM25 retrieval with source attribution.
+
+Wraps ``KnowledgeStore`` so agents can search ingested documents by text query
+and optional provenance filters (source, doc_type, author, date range).
+"""
+
+from __future__ import annotations
+
+from typing import Any, Optional
+
+from openjarvis.connectors.store import KnowledgeStore
+from openjarvis.core.registry import ToolRegistry
+from openjarvis.core.types import ToolResult
+from openjarvis.tools._stubs import BaseTool, ToolSpec
+
+
[email protected]("knowledge_search")
+class KnowledgeSearchTool(BaseTool):
+ """Search the knowledge store using filtered BM25 retrieval.
+
+ Results include source attribution so agents can cite provenance.
+ """
+
+ tool_id = "knowledge_search"
+
+ def __init__(self, store: Optional[KnowledgeStore] = None) -> None:
+ self._store = store
+
+ @property
+ def spec(self) -> ToolSpec:
+ return ToolSpec(
+ name="knowledge_search",
+ description=(
+ "Search ingested personal knowledge (emails, Slack messages,"
+ " documents) using full-text BM25 retrieval with optional"
+ " filters for source, type, author, and date range."
+ ),
+ parameters={
+ "type": "object",
+ "properties": {
+ "query": {
+ "type": "string",
+ "description": "Full-text search query.",
+ },
+ "source": {
+ "type": "string",
+ "description": (
+ "Filter by source connector"
+ " (e.g. 'gmail', 'slack', 'obsidian')."
+ ),
+ },
+ "doc_type": {
+ "type": "string",
+ "description": (
+ "Filter by document type"
+ " (e.g. 'email', 'message', 'document')."
+ ),
+ },
+ "author": {
+ "type": "string",
+ "description": "Filter by author.",
+ },
+ "since": {
+ "type": "string",
+ "description": (
+ "Exclude documents before this ISO 8601 timestamp."
+ ),
+ },
+ "until": {
+ "type": "string",
+ "description": (
+ "Exclude documents after this ISO 8601 timestamp."
+ ),
+ },
+ "top_k": {
+ "type": "integer",
+ "description": "Maximum number of results (default 10).",
+ },
+ },
+ "required": ["query"],
+ },
+ category="knowledge",
+ )
+
+ def execute(self, **params: Any) -> ToolResult:
+ if self._store is None:
+ return ToolResult(
+ tool_name="knowledge_search",
+ content="No knowledge store configured.",
+ success=False,
+ )
+
+ query: str = params.get("query", "")
+ if not query:
+ return ToolResult(
+ tool_name="knowledge_search",
+ content="No query provided.",
+ success=False,
+ )
+
+ top_k: int = int(params.get("top_k", 10))
+ source: Optional[str] = params.get("source")
+ doc_type: Optional[str] = params.get("doc_type")
+ author: Optional[str] = params.get("author")
+ since: Optional[str] = params.get("since")
+ until: Optional[str] = params.get("until")
+
+ results = self._store.retrieve(
+ query,
+ top_k=top_k,
+ source=source,
+ doc_type=doc_type,
+ author=author,
+ since=since,
+ until=until,
+ )
+
+ if not results:
+ return ToolResult(
+ tool_name="knowledge_search",
+ content="No relevant results found.",
+ success=True,
+ metadata={"num_results": 0},
+ )
+
+ lines: list[str] = []
+ for i, result in enumerate(results, start=1):
+ meta = result.metadata
+ src_label = result.source or meta.get("source", "")
+ title = meta.get("title", "")
+ result_author = meta.get("author", "")
+ url = meta.get("url", "")
+
+ # Build header line
+ header_parts: list[str] = []
+ if src_label:
+ header_parts.append(f"[{src_label}]")
+ if title:
+ header_parts.append(title)
+ if result_author:
+ header_parts.append(f"by {result_author}")
+ if url:
+ header_parts.append(f"({url})")
+
+ header = " ".join(header_parts) if header_parts else "(unknown source)"
+ lines.append(f"**Result {i}:** {header}")
+ lines.append(result.content)
+ lines.append("")
+
+ formatted = "\n".join(lines).rstrip()
+
+ return ToolResult(
+ tool_name="knowledge_search",
+ content=formatted,
+ success=True,
+ metadata={"num_results": len(results)},
+ )
+
+
+__all__ = ["KnowledgeSearchTool"]
diff --git a/tests/tools/test_knowledge_search.py b/tests/tools/test_knowledge_search.py
new file mode 100644
index 00000000..e6b32e50
--- /dev/null
+++ b/tests/tools/test_knowledge_search.py
@@ -0,0 +1,139 @@
+"""Tests for the knowledge_search tool."""
+
+from __future__ import annotations
+
+import importlib
+import sys
+
+import pytest
+
+from openjarvis.connectors.store import KnowledgeStore
+from openjarvis.core.registry import ToolRegistry
+from openjarvis.tools.knowledge_search import KnowledgeSearchTool
+
+# ---------------------------------------------------------------------------
+# Fixture
+# ---------------------------------------------------------------------------
+
+
+def store(tmp_path):
+ """Return a KnowledgeStore pre-loaded with 3 diverse items."""
+ s = KnowledgeStore(db_path=tmp_path / "test_knowledge.db")
+
+ # Item 1 — gmail email from alice
+ s.store(
+ "Meeting about Kubernetes migration scheduled for next Tuesday.",
+ source="gmail",
+ doc_type="email",
+ title="Re: K8s migration",
+ author="[email protected]",
+ url="https://mail.google.com/mail/u/0/#inbox/abc123",
+ timestamp="2026-01-15T10:00:00Z",
+ )
+
+ # Item 2 — slack message from bob
+ s.store(
+ "Discussion about K8s costs — we should consider spot instances.",
+ source="slack",
+ doc_type="message",
+ title="#infrastructure",
+ author="[email protected]",
+ url="slack://thread/def456",
+ timestamp="2026-01-20T14:30:00Z",
+ )
+
+ # Item 3 — obsidian document from sarah
+ s.store(
+ "Research notes on large language model fine-tuning strategies.",
+ source="obsidian",
+ doc_type="document",
+ title="LLM Fine-tuning Notes",
+ author="[email protected]",
+ url="obsidian://vault/llm-notes",
+ timestamp="2026-02-01T09:00:00Z",
+ )
+
+ yield s
+ s.close()
+
+
+# ---------------------------------------------------------------------------
+# Tests
+# ---------------------------------------------------------------------------
+
+
+class TestKnowledgeSearchTool:
+ def test_basic_search(self, store):
+ """Search finds the matching document from the 3-item store."""
+ tool = KnowledgeSearchTool(store=store)
+ result = tool.execute(query="Kubernetes migration")
+ assert result.success is True
+ assert "Kubernetes" in result.content
+ assert result.metadata["num_results"] >= 1
+
+ def test_filter_by_source(self, store):
+ """source filter restricts results to gmail only."""
+ tool = KnowledgeSearchTool(store=store)
+ result = tool.execute(query="Kubernetes", source="gmail")
+ assert result.success is True
+ assert "gmail" in result.content
+ # Every returned result must come from gmail
+ assert "slack" not in result.content
+
+ def test_filter_by_author(self, store):
+ """author filter restricts results to sarah only."""
+ tool = KnowledgeSearchTool(store=store)
+ # Use "language model" — FTS5 does not tokenise hyphenated terms like
+ # "fine-tuning" as a single token, so we search for a phrase that works.
+ result = tool.execute(query="language model", author="[email protected]")
+ assert result.success is True
+ assert "[email protected]" in result.content
+ assert result.metadata["num_results"] >= 1
+
+ def test_no_results(self, store):
+ """Query matching nothing returns success=True with 'No relevant results'."""
+ tool = KnowledgeSearchTool(store=store)
+ result = tool.execute(query="zzz_nonexistent_xyzzy_12345")
+ assert result.success is True
+ assert "No relevant results" in result.content
+ assert result.metadata["num_results"] == 0
+
+ def test_empty_query(self, store):
+ """Empty query string returns success=False."""
+ tool = KnowledgeSearchTool(store=store)
+ result = tool.execute(query="")
+ assert result.success is False
+ assert "No query provided" in result.content
+
+ def test_no_store(self):
+ """Missing store returns success=False."""
+ tool = KnowledgeSearchTool()
+ result = tool.execute(query="kubernetes")
+ assert result.success is False
+ assert "No knowledge store configured" in result.content
+
+ def test_spec_has_filter_params(self):
+ """ToolSpec.parameters includes all required and optional filter fields."""
+ tool = KnowledgeSearchTool()
+ props = tool.spec.parameters.get("properties", {})
+ for field in ("query", "source", "doc_type", "author", "since", "top_k"):
+ assert field in props, f"Missing parameter: {field}"
+ assert "query" in tool.spec.parameters.get("required", [])
+ assert tool.spec.category == "knowledge"
+
+ def test_registry(self):
+ """ToolRegistry contains 'knowledge_search' after module import.
+
+ The autouse ``_clean_registries`` fixture clears all registries before
+ each test. Since the module is already cached in ``sys.modules`` a
+ plain import won't re-execute the ``@ToolRegistry.register`` decorator,
+ so we explicitly reload the module.
+ """
+ mod_name = "openjarvis.tools.knowledge_search"
+ if mod_name in sys.modules:
+ importlib.reload(sys.modules[mod_name])
+ else:
+ importlib.import_module(mod_name)
+
+ assert ToolRegistry.contains("knowledge_search")