Personal AI, On Personal Devices
Python
feat: add Apple Notes connector with gzipped HTML extraction
krypticmouseb4a1e824d801
diff --git a/src/openjarvis/connectors/__init__.py b/src/openjarvis/connectors/__init__.pyindex 4baaf5d9..e44b7e02 100644--- a/src/openjarvis/connectors/__init__.py+++ b/src/openjarvis/connectors/__init__.py@@ -42,3 +42,13 @@ try:import openjarvis.connectors.imessage # noqa: F401except ImportError:pass++try:+ import openjarvis.connectors.apple_notes # noqa: F401+except ImportError:+ pass++try:+ import openjarvis.connectors.dropbox # noqa: F401+except ImportError:+ pass # httpx may not be installeddiff --git a/src/openjarvis/connectors/apple_notes.py b/src/openjarvis/connectors/apple_notes.pynew file mode 100644index 00000000..1755a786--- /dev/null+++ b/src/openjarvis/connectors/apple_notes.py@@ -0,0 +1,266 @@+"""Apple Notes connector — reads directly from the macOS Notes SQLite database.++No API calls, no OAuth. The connector opens+``~/Library/Group Containers/group.com.apple.notes/NoteStore.sqlite``+in read-only mode and yields one :class:`Document` per note.++Requires **Full Disk Access** granted to the terminal / app in+System Settings → Privacy & Security → Full Disk Access.++Timestamp notes+---------------+The Notes database stores modification timestamps as seconds since the Apple+epoch of 2001-01-01 00:00:00 UTC. Conversion formula::++ dt = datetime(2001, 1, 1, tzinfo=utc) + timedelta(seconds=ZMODIFICATIONDATE)++Content extraction+------------------+The ``ZDATA`` column in ``ZICNOTEDATA`` contains gzip-compressed HTML. Plain+text is obtained by decompressing the bytes and stripping HTML tags.+"""++from __future__ import annotations++import gzip+import re+import sqlite3+from datetime import datetime, timedelta, timezone+from pathlib import Path+from typing import Iterator, List, Optional++from openjarvis.connectors._stubs import BaseConnector, Document, SyncStatus+from openjarvis.core.registry import ConnectorRegistry+from openjarvis.tools._stubs import ToolSpec++# ---------------------------------------------------------------------------+# Constants+# ---------------------------------------------------------------------------++_DEFAULT_DB_PATH = (+ Path.home()+ / "Library"+ / "Group Containers"+ / "group.com.apple.notes"+ / "NoteStore.sqlite"+)++# Apple epoch: 2001-01-01 00:00:00 UTC+_APPLE_EPOCH = datetime(2001, 1, 1, tzinfo=timezone.utc)++# ---------------------------------------------------------------------------+# Helpers+# ---------------------------------------------------------------------------+++def _apple_ts_to_datetime(apple_seconds: float) -> datetime:+ """Convert an Apple seconds timestamp to a UTC :class:`datetime`.++ Parameters+ ----------+ apple_seconds:+ Seconds since 2001-01-01 00:00:00 UTC.++ Returns+ -------+ datetime+ UTC-aware datetime.+ """+ return _APPLE_EPOCH + timedelta(seconds=apple_seconds)+++def _extract_text_from_zdata(zdata: bytes) -> str:+ """Decompress gzip bytes and strip HTML tags to return plain text.++ Parameters+ ----------+ zdata:+ Raw bytes from the ``ZDATA`` column (gzip-compressed HTML).++ Returns+ -------+ str+ Plain text with all HTML tags removed. Returns an empty string if+ decompression fails.+ """+ try:+ html = gzip.decompress(zdata).decode("utf-8", errors="replace")+ except Exception: # noqa: BLE001+ return ""+ return re.sub(r"<[^>]+>", "", html)+++# ---------------------------------------------------------------------------+# AppleNotesConnector+# ---------------------------------------------------------------------------++[email protected]("apple_notes")+class AppleNotesConnector(BaseConnector):+ """Connector that reads notes from the macOS Notes SQLite database.++ Parameters+ ----------+ db_path:+ Path to ``NoteStore.sqlite``. Defaults to+ ``~/Library/Group Containers/group.com.apple.notes/NoteStore.sqlite``.+ """++ connector_id = "apple_notes"+ display_name = "Apple Notes"+ auth_type = "local"++ def __init__(self, db_path: str = "") -> None:+ self._db_path: Path = Path(db_path) if db_path else _DEFAULT_DB_PATH+ self._connected: bool = False+ self._items_synced: int = 0+ self._items_total: int = 0+ self._last_sync: Optional[datetime] = None++ # ------------------------------------------------------------------+ # BaseConnector interface+ # ------------------------------------------------------------------++ def is_connected(self) -> bool:+ """Return ``True`` if NoteStore.sqlite exists at the configured path."""+ return self._db_path.exists()++ def disconnect(self) -> None:+ """Mark the connector as disconnected."""+ self._connected = False++ def sync(+ self,+ *,+ since: Optional[datetime] = None,+ cursor: Optional[str] = None, # noqa: ARG002+ ) -> Iterator[Document]:+ """Read notes from NoteStore.sqlite and yield one :class:`Document` each.++ Parameters+ ----------+ since:+ If provided, skip notes whose modification time is before this+ datetime.+ cursor:+ Not used for this local connector (included for API+ compatibility).++ Yields+ ------+ Document+ One document per note, with gzip-decompressed plain-text content.+ """+ db_path = str(self._db_path)++ try:+ conn = sqlite3.connect(f"file:{db_path}?mode=ro", uri=True)+ except sqlite3.OperationalError:+ return++ try:+ rows = conn.execute(+ "SELECT n.ZIDENTIFIER, n.ZTITLE, n.ZMODIFICATIONDATE, d.ZDATA "+ "FROM ZICCLOUDSYNCINGOBJECT n "+ "JOIN ZICNOTEDATA d ON d.ZNOTE = n.Z_PK "+ "ORDER BY n.ZMODIFICATIONDATE ASC"+ ).fetchall()++ self._items_total = len(rows)+ synced = 0++ for row in rows:+ identifier: str = row[0] or ""+ title: str = row[1] or ""+ mod_date: float = row[2] or 0.0+ zdata: bytes = row[3] or b""++ timestamp = _apple_ts_to_datetime(mod_date)++ # Apply since filter+ if since is not None:+ since_utc = since+ if since_utc.tzinfo is None:+ since_utc = since_utc.replace(tzinfo=timezone.utc)+ if timestamp < since_utc:+ continue++ content = _extract_text_from_zdata(zdata)++ doc = Document(+ doc_id=f"apple_notes:{identifier}",+ source="apple_notes",+ doc_type="note",+ content=content,+ title=title,+ timestamp=timestamp,+ )+ synced += 1+ yield doc++ self._items_synced = synced+ self._last_sync = datetime.now(tz=timezone.utc)++ finally:+ conn.close()++ def sync_status(self) -> SyncStatus:+ """Return sync progress from the most recent :meth:`sync` call."""+ return SyncStatus(+ state="idle",+ items_synced=self._items_synced,+ items_total=self._items_total,+ last_sync=self._last_sync,+ )++ # ------------------------------------------------------------------+ # MCP tools+ # ------------------------------------------------------------------++ def mcp_tools(self) -> List[ToolSpec]:+ """Expose two MCP tool specs for real-time Apple Notes queries."""+ return [+ ToolSpec(+ name="notes_search",+ description=(+ "Search Apple Notes by keyword. "+ "Returns matching notes with title and content snippet."+ ),+ parameters={+ "type": "object",+ "properties": {+ "query": {+ "type": "string",+ "description": "Search query string",+ },+ "max_results": {+ "type": "integer",+ "description": "Maximum number of notes to return",+ "default": 20,+ },+ },+ "required": ["query"],+ },+ category="knowledge",+ ),+ ToolSpec(+ name="notes_get_note",+ description=(+ "Retrieve the full text content of an Apple Note by its "+ "unique identifier."+ ),+ parameters={+ "type": "object",+ "properties": {+ "note_id": {+ "type": "string",+ "description": (+ "Apple Notes unique identifier (ZIDENTIFIER)"+ ),+ },+ },+ "required": ["note_id"],+ },+ category="knowledge",+ ),+ ]diff --git a/tests/connectors/test_apple_notes.py b/tests/connectors/test_apple_notes.pynew file mode 100644index 00000000..c3ee4ed5--- /dev/null+++ b/tests/connectors/test_apple_notes.py@@ -0,0 +1,209 @@+"""Tests for AppleNotesConnector — local macOS Notes database connector.++All tests use a temporary SQLite database that mimics the real NoteStore.sqlite+schema. No actual macOS Notes database is required.+"""++from __future__ import annotations++import gzip+import sqlite3+from pathlib import Path+from typing import List++import pytest++from openjarvis.connectors._stubs import Document+from openjarvis.core.registry import ConnectorRegistry++# ---------------------------------------------------------------------------+# Helper: create a fake NoteStore.sqlite+# ---------------------------------------------------------------------------+++def _create_fake_notes_db(db_path: Path) -> None:+ """Populate a SQLite file with the Apple Notes schema and sample rows."""+ conn = sqlite3.connect(str(db_path))+ conn.executescript("""+ CREATE TABLE ZICCLOUDSYNCINGOBJECT (+ Z_PK INTEGER PRIMARY KEY,+ ZTITLE TEXT,+ ZMODIFICATIONDATE REAL,+ ZIDENTIFIER TEXT,+ ZNOTE INTEGER+ );+ CREATE TABLE ZICNOTEDATA (+ Z_PK INTEGER PRIMARY KEY,+ ZDATA BLOB,+ ZNOTE INTEGER+ );+ """)++ # Note 1 — Shopping List+ html1 = "<html><body><h1>Shopping List</h1><p>Milk, eggs, bread</p></body></html>"+ compressed1 = gzip.compress(html1.encode())+ conn.execute(+ "INSERT INTO ZICCLOUDSYNCINGOBJECT VALUES "+ "(1, 'Shopping List', 694310400.0, 'note-001', 1)"+ )+ conn.execute("INSERT INTO ZICNOTEDATA VALUES (1, ?, 1)", (compressed1,))++ # Note 2 — Meeting Notes+ html2 = "<html><body><p>Meeting notes from Monday</p></body></html>"+ compressed2 = gzip.compress(html2.encode())+ conn.execute(+ "INSERT INTO ZICCLOUDSYNCINGOBJECT VALUES "+ "(2, 'Meeting Notes', 694396800.0, 'note-002', 2)"+ )+ conn.execute("INSERT INTO ZICNOTEDATA VALUES (2, ?, 2)", (compressed2,))++ conn.commit()+ conn.close()+++# ---------------------------------------------------------------------------+# Fixtures+# ---------------------------------------------------------------------------+++def fake_db(tmp_path: Path) -> Path:+ """Return path to a populated fake NoteStore.sqlite."""+ db_path = tmp_path / "NoteStore.sqlite"+ _create_fake_notes_db(db_path)+ return db_path+++def connector(fake_db: Path):+ """AppleNotesConnector pointing at the fake DB."""+ from openjarvis.connectors.apple_notes import AppleNotesConnector # noqa: PLC0415++ return AppleNotesConnector(db_path=str(fake_db))+++# ---------------------------------------------------------------------------+# Test 1 — is_connected returns True when db_path exists+# ---------------------------------------------------------------------------+++def test_is_connected(connector) -> None:+ """is_connected() returns True when NoteStore.sqlite exists."""+ assert connector.is_connected() is True+++# ---------------------------------------------------------------------------+# Test 2 — is_connected returns False for a missing file+# ---------------------------------------------------------------------------+++def test_not_connected_missing_db() -> None:+ """is_connected() returns False when the database file does not exist."""+ from openjarvis.connectors.apple_notes import AppleNotesConnector # noqa: PLC0415++ conn = AppleNotesConnector(db_path="/nonexistent/path/NoteStore.sqlite")+ assert conn.is_connected() is False+++# ---------------------------------------------------------------------------+# Test 3 — sync yields 2 notes with correct source and doc_type+# ---------------------------------------------------------------------------+++def test_sync_yields_notes(connector) -> None:+ """sync() yields one Document per note (2 total)."""+ docs: List[Document] = list(connector.sync())+ assert len(docs) == 2+ for doc in docs:+ assert doc.source == "apple_notes"+ assert doc.doc_type == "note"+++# ---------------------------------------------------------------------------+# Test 4 — sync decompresses content and strips HTML tags+# ---------------------------------------------------------------------------+++def test_sync_decompresses_content(connector) -> None:+ """sync() produces plain text with no HTML tags in content."""+ docs: List[Document] = list(connector.sync())++ for doc in docs:+ # No HTML tags should remain+ assert "<" not in doc.content+ assert ">" not in doc.content++ # Content should include the text from the HTML bodies+ contents = {d.content for d in docs}+ combined = " ".join(contents)+ assert "Shopping List" in combined or "Milk" in combined or "eggs" in combined+ assert "Meeting notes" in combined+++# ---------------------------------------------------------------------------+# Test 5 — sync sets doc_type to "note"+# ---------------------------------------------------------------------------+++def test_sync_sets_doc_type_note(connector) -> None:+ """All documents yielded by sync() have doc_type == 'note'."""+ docs: List[Document] = list(connector.sync())+ for doc in docs:+ assert doc.doc_type == "note"+++# ---------------------------------------------------------------------------+# Test 6 — sync sets correct doc_id and title+# ---------------------------------------------------------------------------+++def test_sync_doc_ids_and_titles(connector) -> None:+ """sync() sets doc_id from ZIDENTIFIER and title from ZTITLE."""+ docs: List[Document] = list(connector.sync())++ doc1 = next(d for d in docs if d.doc_id == "apple_notes:note-001")+ assert doc1.title == "Shopping List"++ doc2 = next(d for d in docs if d.doc_id == "apple_notes:note-002")+ assert doc2.title == "Meeting Notes"+++# ---------------------------------------------------------------------------+# Test 7 — disconnect sets connected flag to False+# ---------------------------------------------------------------------------+++def test_disconnect(connector) -> None:+ """disconnect() marks the connector as disconnected."""+ assert connector.is_connected() is True+ connector.disconnect()+ assert connector._connected is False+++# ---------------------------------------------------------------------------+# Test 8 — mcp_tools returns exactly 2 tool specs+# ---------------------------------------------------------------------------+++def test_mcp_tools(connector) -> None:+ """mcp_tools() returns exactly 2 tools with the expected names."""+ tools = connector.mcp_tools()+ names = {t.name for t in tools}+ assert len(tools) == 2+ assert "notes_search" in names+ assert "notes_get_note" in names+++# ---------------------------------------------------------------------------+# Test 9 — ConnectorRegistry contains "apple_notes" after import+# ---------------------------------------------------------------------------+++def test_registry() -> None:+ """AppleNotesConnector is registered and retrievable via ConnectorRegistry."""+ from openjarvis.connectors.apple_notes import AppleNotesConnector # noqa: PLC0415++ ConnectorRegistry.register_value("apple_notes", AppleNotesConnector)+ assert ConnectorRegistry.contains("apple_notes")+ cls = ConnectorRegistry.get("apple_notes")+ assert cls.connector_id == "apple_notes"