Personal AI, On Personal Devices
Python
feat: add Obsidian/Markdown vault connector with frontmatter parsing
krypticmouse5e0e8965d75e
diff --git a/src/openjarvis/connectors/obsidian.py b/src/openjarvis/connectors/obsidian.pynew file mode 100644index 00000000..1625d146--- /dev/null+++ b/src/openjarvis/connectors/obsidian.py@@ -0,0 +1,235 @@+"""Obsidian / Markdown vault connector.++Reads ``.md``, ``.markdown``, and ``.txt`` files from a local vault directory,+parses optional YAML frontmatter, and yields :class:`Document` objects that+can be ingested by the knowledge pipeline.+"""++from __future__ import annotations++import os+from datetime import datetime+from pathlib import Path+from typing import Any, Dict, Iterator, List, Optional, Tuple+from urllib.parse import quote++from openjarvis.connectors._stubs import BaseConnector, Document, SyncStatus+from openjarvis.core.registry import ConnectorRegistry+from openjarvis.tools._stubs import ToolSpec++# ---------------------------------------------------------------------------+# Constants+# ---------------------------------------------------------------------------++_TEXT_EXTENSIONS = {".md", ".markdown", ".txt"}++_SKIP_DIRS = {+ ".obsidian",+ ".git",+ ".trash",+ "__pycache__",+ "node_modules",+ ".venv",+}++# ---------------------------------------------------------------------------+# Frontmatter parser (no PyYAML dependency)+# ---------------------------------------------------------------------------+++def _parse_frontmatter(text: str) -> Tuple[Dict[str, Any], str]:+ """Extract YAML frontmatter and return ``(metadata, body)``.++ Frontmatter is the block between two ``---`` lines at the very start of+ the file. Only simple ``key: value`` and ``key: [a, b, c]`` syntax is+ handled — no nested YAML.+ """+ if not text.startswith("---"):+ return {}, text++ # Find the closing --- marker+ rest = text[3:]+ # Accept both \n--- and \r\n---+ end_marker = "\n---"+ end_idx = rest.find(end_marker)+ if end_idx == -1:+ return {}, text++ raw_fm = rest[:end_idx]+ # Body starts after the closing ---+ body_start = end_idx + len(end_marker)+ # Consume an optional trailing newline after the closing ---+ if body_start < len(rest) and rest[body_start] == "\n":+ body_start += 1+ body = rest[body_start:]++ metadata: Dict[str, Any] = {}+ for line in raw_fm.splitlines():+ line = line.strip()+ if not line or ":" not in line:+ continue+ key, _, raw_value = line.partition(":")+ key = key.strip()+ raw_value = raw_value.strip()+ if not key:+ continue++ # List syntax: [a, b, c]+ if raw_value.startswith("[") and raw_value.endswith("]"):+ inner = raw_value[1:-1]+ items = [v.strip().strip("'\"") for v in inner.split(",") if v.strip()]+ metadata[key] = items+ else:+ # Strip optional surrounding quotes+ metadata[key] = raw_value.strip("'\"")++ return metadata, body+++# ---------------------------------------------------------------------------+# ObsidianConnector+# ---------------------------------------------------------------------------++[email protected]("obsidian")+class ObsidianConnector(BaseConnector):+ """Connector that reads a local Obsidian (or plain Markdown) vault.++ Parameters+ ----------+ vault_path:+ Absolute path to the vault root directory. An empty string means+ "not yet configured" — :meth:`is_connected` will return ``False``.+ """++ connector_id = "obsidian"+ display_name = "Obsidian / Markdown"+ auth_type = "filesystem"++ def __init__(self, vault_path: str = "") -> None:+ self._vault_path = vault_path+ self._connected: bool = bool(vault_path) and Path(vault_path).is_dir()+ self._items_synced: int = 0+ self._items_total: int = 0++ # ------------------------------------------------------------------+ # BaseConnector interface+ # ------------------------------------------------------------------++ def is_connected(self) -> bool:+ """Return ``True`` if vault_path is set and the directory exists."""+ return bool(self._vault_path) and Path(self._vault_path).is_dir()++ def disconnect(self) -> None:+ """Clear vault_path and mark as disconnected."""+ self._vault_path = ""+ self._connected = False++ def sync(+ self,+ *,+ since: Optional[datetime] = None,+ cursor: Optional[str] = None, # noqa: ARG002 — unused but part of ABC+ ) -> Iterator[Document]:+ """Walk the vault and yield one :class:`Document` per text file.++ Parameters+ ----------+ since:+ If provided, skip files whose mtime is before this datetime.+ cursor:+ Not used for this filesystem connector (included for API+ compatibility).+ """+ vault = Path(self._vault_path)+ vault_name = vault.name++ collected_paths: List[Path] = []+ for root, dirs, files in os.walk(vault):+ # Prune hidden and known-junk directories in-place so os.walk+ # does not descend into them.+ dirs[:] = [d for d in dirs if d not in _SKIP_DIRS and not d.startswith(".")]++ for filename in files:+ fpath = Path(root) / filename+ if fpath.suffix.lower() not in _TEXT_EXTENSIONS:+ continue+ collected_paths.append(fpath)++ self._items_total = len(collected_paths)+ synced = 0++ for fpath in collected_paths:+ # Apply since filter based on mtime+ mtime = datetime.fromtimestamp(fpath.stat().st_mtime)+ if since is not None and mtime < since:+ continue++ try:+ text = fpath.read_text(encoding="utf-8", errors="replace")+ except (OSError, PermissionError):+ continue++ metadata, _body = _parse_frontmatter(text)++ title = metadata.get("title") or fpath.stem+ rel_path = fpath.relative_to(vault)++ url = (+ f"obsidian://open?vault={quote(vault_name)}&file={quote(str(rel_path))}"+ )++ doc = Document(+ doc_id=f"obsidian:{rel_path}",+ source="obsidian",+ doc_type="note",+ content=text,+ title=str(title),+ timestamp=mtime,+ url=url,+ metadata={k: v for k, v in metadata.items() if k != "title"},+ )+ synced += 1+ yield doc++ self._items_synced = synced++ def sync_status(self) -> SyncStatus:+ """Return sync progress from the most recent :meth:`sync` call."""+ return SyncStatus(+ state="idle",+ items_synced=self._items_synced,+ items_total=self._items_total,+ )++ # ------------------------------------------------------------------+ # MCP tools+ # ------------------------------------------------------------------++ def mcp_tools(self) -> List[ToolSpec]:+ """Expose a single ``obsidian_search_notes`` tool for agent queries."""+ return [+ ToolSpec(+ name="obsidian_search_notes",+ description=(+ "Search notes in the local Obsidian vault by keyword. "+ "Returns matching note titles and snippets."+ ),+ parameters={+ "type": "object",+ "properties": {+ "query": {+ "type": "string",+ "description": "Search query string",+ },+ "top_k": {+ "type": "integer",+ "description": "Maximum number of results to return",+ "default": 10,+ },+ },+ "required": ["query"],+ },+ category="knowledge",+ )+ ]diff --git a/tests/connectors/test_obsidian.py b/tests/connectors/test_obsidian.pynew file mode 100644index 00000000..7d4e519f--- /dev/null+++ b/tests/connectors/test_obsidian.py@@ -0,0 +1,186 @@+"""Tests for ObsidianConnector — filesystem Markdown vault connector."""++from __future__ import annotations++from pathlib import Path+from typing import List++import pytest++from openjarvis.connectors._stubs import Document+from openjarvis.core.registry import ConnectorRegistry++# ---------------------------------------------------------------------------+# Fixture: small vault on disk+# ---------------------------------------------------------------------------+++def vault(tmp_path: Path) -> Path:+ """Build a minimal Obsidian vault under *tmp_path*."""+ # note1.md — has frontmatter with title and tags+ (tmp_path / "note1.md").write_text(+ "---\n"+ "title: My First Note\n"+ "tags: [research, ai]\n"+ "---\n"+ "# My First Note\n\n"+ "This is the body of the first note.\n",+ encoding="utf-8",+ )++ # note2.md — no frontmatter+ (tmp_path / "note2.md").write_text(+ "# Plain Note\n\nNo frontmatter here.\n",+ encoding="utf-8",+ )++ # subdir/deep.md — nested note+ subdir = tmp_path / "subdir"+ subdir.mkdir()+ (subdir / "deep.md").write_text(+ "# Deep Note\n\nNested file content.\n",+ encoding="utf-8",+ )++ # .obsidian/config.json — should be skipped (hidden dir)+ hidden = tmp_path / ".obsidian"+ hidden.mkdir()+ (hidden / "config.json").write_text('{"theme": "dark"}\n', encoding="utf-8")++ # image.png — should be skipped (binary/non-text extension)+ (tmp_path / "image.png").write_bytes(b"\x89PNG\r\n\x1a\n\x00\x00\x00\rIHDR")++ return tmp_path+++# ---------------------------------------------------------------------------+# Helper+# ---------------------------------------------------------------------------+++def _sync_all(vault_path: Path) -> List[Document]:+ """Import connector (triggers registry) and collect all docs."""+ from openjarvis.connectors.obsidian import ObsidianConnector # noqa: PLC0415++ conn = ObsidianConnector(vault_path=str(vault_path))+ return list(conn.sync())+++# ---------------------------------------------------------------------------+# Test 1: is_connected — True for a valid directory+# ---------------------------------------------------------------------------+++def test_is_connected(vault: Path) -> None:+ from openjarvis.connectors.obsidian import ObsidianConnector++ conn = ObsidianConnector(vault_path=str(vault))+ assert conn.is_connected() is True+++# ---------------------------------------------------------------------------+# Test 2: not_connected_bad_path — False for a nonexistent path+# ---------------------------------------------------------------------------+++def test_not_connected_bad_path() -> None:+ from openjarvis.connectors.obsidian import ObsidianConnector++ conn = ObsidianConnector(vault_path="/nonexistent/path/that/does/not/exist")+ assert conn.is_connected() is False+++# ---------------------------------------------------------------------------+# Test 3: sync_yields_markdown_files — correct count including subdirs+# ---------------------------------------------------------------------------+++def test_sync_yields_markdown_files(vault: Path) -> None:+ """sync() yields all .md files including those in subdirectories."""+ docs = _sync_all(vault)+ # note1.md, note2.md, subdir/deep.md → 3 documents+ # .obsidian/config.json and image.png must NOT appear+ assert len(docs) == 3+++# ---------------------------------------------------------------------------+# Test 4: sync_skips_hidden_dirs — .obsidian dir content not included+# ---------------------------------------------------------------------------+++def test_sync_skips_hidden_dirs(vault: Path) -> None:+ """Files inside .obsidian (and other hidden dirs) are not yielded."""+ docs = _sync_all(vault)+ rel_paths = [d.url or "" for d in docs]+ assert not any(".obsidian" in p for p in rel_paths)+ # config.json should never appear in doc titles either+ assert not any(d.title == "config" for d in docs)+++# ---------------------------------------------------------------------------+# Test 5: sync_skips_binary_files — .png files not included+# ---------------------------------------------------------------------------+++def test_sync_skips_binary_files(vault: Path) -> None:+ """Non-text files (.png, .jpg, etc.) are skipped entirely."""+ docs = _sync_all(vault)+ assert not any(d.doc_id.endswith(".png") for d in docs)+ assert not any("image" in d.doc_id for d in docs)+++# ---------------------------------------------------------------------------+# Test 6: sync_parses_frontmatter — title and tags extracted+# ---------------------------------------------------------------------------+++def test_sync_parses_frontmatter(vault: Path) -> None:+ """note1.md frontmatter title is used; tags land in metadata."""+ docs = _sync_all(vault)+ note1 = next(d for d in docs if "note1" in d.doc_id)+ assert note1.title == "My First Note"+ assert note1.metadata.get("tags") == ["research", "ai"]+++# ---------------------------------------------------------------------------+# Test 7: sync_sets_doc_type_note — all docs have doc_type="note", source="obsidian"+# ---------------------------------------------------------------------------+++def test_sync_sets_doc_type_note(vault: Path) -> None:+ """Every yielded document has doc_type='note' and source='obsidian'."""+ docs = _sync_all(vault)+ assert len(docs) > 0+ for doc in docs:+ assert doc.doc_type == "note"+ assert doc.source == "obsidian"+++# ---------------------------------------------------------------------------+# Test 8: disconnect — sets is_connected to False+# ---------------------------------------------------------------------------+++def test_disconnect(vault: Path) -> None:+ from openjarvis.connectors.obsidian import ObsidianConnector++ conn = ObsidianConnector(vault_path=str(vault))+ assert conn.is_connected() is True+ conn.disconnect()+ assert conn.is_connected() is False+++# ---------------------------------------------------------------------------+# Test 9: registry — ConnectorRegistry.contains("obsidian") after import+# ---------------------------------------------------------------------------+++def test_registry() -> None:+ """ObsidianConnector can be registered and retrieved via ConnectorRegistry."""+ from openjarvis.connectors.obsidian import ObsidianConnector # noqa: PLC0415++ ConnectorRegistry.register_value("obsidian", ObsidianConnector)+ assert ConnectorRegistry.contains("obsidian")+ cls = ConnectorRegistry.get("obsidian")+ assert cls.connector_id == "obsidian"