Personal AI, On Personal Devices
Python
feat: add Notion connector with block-to-markdown rendering
krypticmouse547b6ba734fd
diff --git a/src/openjarvis/connectors/__init__.py b/src/openjarvis/connectors/__init__.pyindex 9e935c8e..32e5458d 100644--- a/src/openjarvis/connectors/__init__.py+++ b/src/openjarvis/connectors/__init__.py@@ -17,3 +17,8 @@ try:import openjarvis.connectors.gmail # noqa: F401except ImportError:pass # httpx may not be installed++try:+ import openjarvis.connectors.notion # noqa: F401+except ImportError:+ passdiff --git a/src/openjarvis/connectors/notion.py b/src/openjarvis/connectors/notion.pynew file mode 100644index 00000000..e8104ef1--- /dev/null+++ b/src/openjarvis/connectors/notion.py@@ -0,0 +1,441 @@+"""Notion connector — syncs pages via the Notion REST API.++Uses a Notion internal integration token stored locally. All network calls+are isolated in module-level functions (``_notion_api_*``) to make them+trivially mockable in tests.++Users create an internal integration at https://www.notion.so/my-integrations+and paste the generated token to authenticate.+"""++from __future__ import annotations++from datetime import datetime, timezone+from typing import Any, Dict, Iterator, List, Optional++import httpx++from openjarvis.connectors._stubs import BaseConnector, Document, SyncStatus+from openjarvis.connectors.oauth import delete_tokens, load_tokens, save_tokens+from openjarvis.core.config import DEFAULT_CONFIG_DIR+from openjarvis.core.registry import ConnectorRegistry+from openjarvis.tools._stubs import ToolSpec++# ---------------------------------------------------------------------------+# Constants+# ---------------------------------------------------------------------------++_NOTION_API_BASE = "https://api.notion.com/v1"+_NOTION_VERSION = "2022-06-28"+_DEFAULT_CREDENTIALS_PATH = str(+ DEFAULT_CONFIG_DIR / "connectors" / "notion.json"+)++# ---------------------------------------------------------------------------+# Module-level API functions (easy to patch in tests)+# ---------------------------------------------------------------------------+++def _notion_headers(token: str) -> Dict[str, str]:+ """Build the standard Notion API request headers."""+ return {+ "Authorization": f"Bearer {token}",+ "Notion-Version": _NOTION_VERSION,+ "Content-Type": "application/json",+ }+++def _notion_api_search(+ token: str,+ *,+ cursor: Optional[str] = None,+) -> Dict[str, Any]:+ """Call the Notion ``/v1/search`` endpoint to list accessible pages.++ Parameters+ ----------+ token:+ Notion integration token.+ cursor:+ Pagination cursor (``start_cursor``) from a previous response.++ Returns+ -------+ dict+ Raw API response containing ``results`` list and ``has_more`` flag.+ """+ body: Dict[str, Any] = {+ "filter": {"property": "object", "value": "page"},+ "page_size": 100,+ }+ if cursor:+ body["start_cursor"] = cursor++ resp = httpx.post(+ f"{_NOTION_API_BASE}/search",+ headers=_notion_headers(token),+ json=body,+ timeout=30.0,+ )+ resp.raise_for_status()+ return resp.json()+++def _notion_api_get_blocks(token: str, block_id: str) -> List[Dict[str, Any]]:+ """Fetch child blocks for a page (or block) by ID.++ Parameters+ ----------+ token:+ Notion integration token.+ block_id:+ Page or block UUID.++ Returns+ -------+ list[dict]+ List of block objects from the API ``results`` field.+ """+ resp = httpx.get(+ f"{_NOTION_API_BASE}/blocks/{block_id}/children",+ headers=_notion_headers(token),+ params={"page_size": 100},+ timeout=30.0,+ )+ resp.raise_for_status()+ return resp.json().get("results", [])+++# ---------------------------------------------------------------------------+# Block-to-markdown renderer+# ---------------------------------------------------------------------------+++def _extract_rich_text(block_data: Dict[str, Any]) -> str:+ """Concatenate plain_text from all rich_text entries in *block_data*."""+ rich_text: List[Dict[str, Any]] = block_data.get("rich_text", [])+ return "".join(entry.get("plain_text", "") for entry in rich_text)+++def _render_blocks_to_markdown(blocks: List[Dict[str, Any]]) -> str:+ """Convert a list of Notion block objects to a markdown string.++ Supported block types:+ - ``paragraph`` → plain text+ - ``heading_1`` → ``# text``+ - ``heading_2`` → ``## text``+ - ``heading_3`` → ``### text``+ - ``bulleted_list_item`` → ``- text``+ - ``numbered_list_item`` → ``1. text``+ - ``to_do`` → ``- [ ] text`` or ``- [x] text``+ - ``code`` → fenced code block+ - ``quote`` → ``> text``+ - ``divider`` → ``---``+ - ``toggle`` → rendered as plain heading + content+ - Everything else → extract rich_text if available, skip otherwise+ """+ lines: List[str] = []++ for block in blocks:+ block_type: str = block.get("type", "")+ block_data: Dict[str, Any] = block.get(block_type, {})++ if block_type == "paragraph":+ text = _extract_rich_text(block_data)+ lines.append(text)++ elif block_type == "heading_1":+ text = _extract_rich_text(block_data)+ lines.append(f"# {text}")++ elif block_type == "heading_2":+ text = _extract_rich_text(block_data)+ lines.append(f"## {text}")++ elif block_type == "heading_3":+ text = _extract_rich_text(block_data)+ lines.append(f"### {text}")++ elif block_type == "bulleted_list_item":+ text = _extract_rich_text(block_data)+ lines.append(f"- {text}")++ elif block_type == "numbered_list_item":+ text = _extract_rich_text(block_data)+ lines.append(f"1. {text}")++ elif block_type == "to_do":+ text = _extract_rich_text(block_data)+ checked: bool = block_data.get("checked", False)+ checkbox = "x" if checked else " "+ lines.append(f"- [{checkbox}] {text}")++ elif block_type == "code":+ text = _extract_rich_text(block_data)+ language: str = block_data.get("language", "")+ lines.append(f"```{language}")+ lines.append(text)+ lines.append("```")++ elif block_type == "quote":+ text = _extract_rich_text(block_data)+ lines.append(f"> {text}")++ elif block_type == "divider":+ lines.append("---")++ elif block_type == "toggle":+ text = _extract_rich_text(block_data)+ lines.append(text)++ else:+ # Generic fallback: extract rich_text if the block data has it+ if "rich_text" in block_data:+ text = _extract_rich_text(block_data)+ if text:+ lines.append(text)++ return "\n".join(lines)+++# ---------------------------------------------------------------------------+# Title extractor+# ---------------------------------------------------------------------------+++def _extract_page_title(page: Dict[str, Any]) -> str:+ """Extract the plain-text title from a Notion page object."""+ properties: Dict[str, Any] = page.get("properties", {})++ # The title property can be named "title", "Name", or any custom name.+ # Notion always marks the title property with type "title".+ for _prop_name, prop_value in properties.items():+ if not isinstance(prop_value, dict):+ continue+ title_items: List[Dict[str, Any]] = prop_value.get("title", [])+ if title_items:+ return "".join(item.get("plain_text", "") for item in title_items)++ return "(Untitled)"+++def _parse_iso_datetime(dt_str: str) -> datetime:+ """Parse an ISO 8601 datetime string (as returned by Notion) into a datetime."""+ if not dt_str:+ return datetime.now(tz=timezone.utc)+ try:+ # Python 3.11+ handles 'Z' directly; for 3.10 replace manually.+ return datetime.fromisoformat(dt_str.replace("Z", "+00:00"))+ except (ValueError, AttributeError):+ return datetime.now(tz=timezone.utc)+++# ---------------------------------------------------------------------------+# NotionConnector+# ---------------------------------------------------------------------------++[email protected]("notion")+class NotionConnector(BaseConnector):+ """Connector that syncs pages from Notion via the REST API.++ Authentication uses a Notion internal integration token. Users create+ an integration at https://www.notion.so/my-integrations and paste the+ generated secret token.++ Parameters+ ----------+ token:+ Notion integration token. If provided, it takes priority over any+ stored credentials file.+ credentials_path:+ Path to the JSON file where the token is stored. Defaults to+ ``~/.openjarvis/connectors/notion.json``.+ """++ connector_id = "notion"+ display_name = "Notion"+ auth_type = "oauth" # token-based, same pattern as OAuth connectors++ def __init__(+ self,+ token: str = "",+ credentials_path: str = "",+ ) -> None:+ self._token: str = token+ self._credentials_path: str = credentials_path or _DEFAULT_CREDENTIALS_PATH+ self._items_synced: int = 0+ self._items_total: int = 0+ self._last_sync: Optional[datetime] = None+ self._last_cursor: Optional[str] = None++ # ------------------------------------------------------------------+ # Internal token resolution+ # ------------------------------------------------------------------++ def _resolve_token(self) -> str:+ """Return the active token — direct > file."""+ if self._token:+ return self._token+ tokens = load_tokens(self._credentials_path)+ if tokens:+ return tokens.get("token", "")+ return ""++ # ------------------------------------------------------------------+ # BaseConnector interface+ # ------------------------------------------------------------------++ def is_connected(self) -> bool:+ """Return ``True`` if a valid token is available."""+ return bool(self._resolve_token())++ def disconnect(self) -> None:+ """Clear the in-memory token and delete the stored credentials file."""+ self._token = ""+ delete_tokens(self._credentials_path)++ def auth_url(self) -> str:+ """Return the URL where users can create a Notion integration token."""+ return "https://www.notion.so/my-integrations"++ def handle_callback(self, code: str) -> None:+ """Persist the integration token to the credentials file.++ The *code* parameter holds the raw token string (pasted by the user).+ """+ save_tokens(self._credentials_path, {"token": code})++ def sync(+ self,+ *,+ since: Optional[datetime] = None,+ cursor: Optional[str] = None,+ ) -> Iterator[Document]:+ """Yield :class:`Document` objects for Notion pages.++ Paginates through ``/v1/search`` and fetches block content for each+ page, rendering it to markdown.++ Parameters+ ----------+ since:+ If provided, skip pages whose ``last_edited_time`` is before this+ datetime.+ cursor:+ ``next_cursor`` from a previous sync to resume pagination.+ """+ token = self._resolve_token()+ if not token:+ return++ page_cursor: Optional[str] = cursor+ synced = 0++ while True:+ search_resp = _notion_api_search(token, cursor=page_cursor)+ pages: List[Dict[str, Any]] = search_resp.get("results", [])++ for page in pages:+ page_id: str = page.get("id", "")+ if not page_id:+ continue++ last_edited_str: str = page.get("last_edited_time", "")+ timestamp = _parse_iso_datetime(last_edited_str)++ # Apply since filter+ if since is not None:+ # Make since timezone-aware for comparison if needed+ since_aware = since+ if since.tzinfo is None and timestamp.tzinfo is not None:+ since_aware = since.replace(tzinfo=timezone.utc)+ if timestamp < since_aware:+ continue++ title = _extract_page_title(page)+ url: Optional[str] = page.get("url")++ blocks = _notion_api_get_blocks(token, page_id)+ content = _render_blocks_to_markdown(blocks)++ doc = Document(+ doc_id=f"notion:{page_id}",+ source="notion",+ doc_type="document",+ content=content,+ title=title,+ timestamp=timestamp,+ url=url,+ metadata={"page_id": page_id},+ )+ synced += 1+ yield doc++ has_more: bool = search_resp.get("has_more", False)+ if not has_more:+ self._last_cursor = None+ break+ page_cursor = search_resp.get("next_cursor")+ self._last_cursor = page_cursor++ self._items_synced = synced+ self._last_sync = datetime.now(tz=timezone.utc)++ def sync_status(self) -> SyncStatus:+ """Return sync progress from the most recent :meth:`sync` call."""+ return SyncStatus(+ state="idle",+ items_synced=self._items_synced,+ last_sync=self._last_sync,+ cursor=self._last_cursor,+ )++ # ------------------------------------------------------------------+ # MCP tools+ # ------------------------------------------------------------------++ def mcp_tools(self) -> List[ToolSpec]:+ """Expose two MCP tool specs for real-time Notion queries."""+ return [+ ToolSpec(+ name="notion_search_pages",+ description=(+ "Search Notion pages accessible to the integration by keyword. "+ "Returns matching page titles and URLs."+ ),+ parameters={+ "type": "object",+ "properties": {+ "query": {+ "type": "string",+ "description": "Search query string",+ },+ "max_results": {+ "type": "integer",+ "description": "Maximum number of pages to return",+ "default": 20,+ },+ },+ "required": ["query"],+ },+ category="knowledge",+ ),+ ToolSpec(+ name="notion_get_page",+ description=(+ "Retrieve the full markdown content of a Notion page by its ID."+ ),+ parameters={+ "type": "object",+ "properties": {+ "page_id": {+ "type": "string",+ "description": "Notion page UUID",+ },+ },+ "required": ["page_id"],+ },+ category="knowledge",+ ),+ ]diff --git a/tests/connectors/test_notion.py b/tests/connectors/test_notion.pynew file mode 100644index 00000000..c9f3a9ae--- /dev/null+++ b/tests/connectors/test_notion.py@@ -0,0 +1,243 @@+"""Tests for NotionConnector — Notion API sync connector.++All Notion API calls are mocked; no network access is required.+"""++from __future__ import annotations++import json+from pathlib import Path+from typing import List+from unittest.mock import patch++import pytest++from openjarvis.connectors._stubs import Document+from openjarvis.core.registry import ConnectorRegistry++# ---------------------------------------------------------------------------+# Fake API payloads+# ---------------------------------------------------------------------------++_SEARCH_RESPONSE = {+ "results": [+ {+ "id": "page1",+ "object": "page",+ "properties": {"title": {"title": [{"plain_text": "Meeting Notes"}]}},+ "last_edited_time": "2024-03-15T10:00:00.000Z",+ "url": "https://notion.so/page1",+ },+ {+ "id": "page2",+ "object": "page",+ "properties": {"title": {"title": [{"plain_text": "Project Plan"}]}},+ "last_edited_time": "2024-03-16T11:00:00.000Z",+ "url": "https://notion.so/page2",+ },+ ],+ "has_more": False,+}++_BLOCKS_RESPONSE = [+ {"type": "paragraph", "paragraph": {"rich_text": [{"plain_text": "Hello world"}]}},+ {"type": "heading_2", "heading_2": {"rich_text": [{"plain_text": "Section"}]}},+]++# ---------------------------------------------------------------------------+# Fixtures+# ---------------------------------------------------------------------------+++def connector(tmp_path: Path):+ """NotionConnector pointing at a tmp credentials path (no file yet)."""+ from openjarvis.connectors.notion import NotionConnector # noqa: PLC0415++ creds_path = str(tmp_path / "notion.json")+ return NotionConnector(credentials_path=creds_path)+++# ---------------------------------------------------------------------------+# Test 1 — not connected without a token or credentials file+# ---------------------------------------------------------------------------+++def test_not_connected_without_token(connector) -> None:+ """is_connected() returns False when no token and no credentials file exist."""+ assert connector.is_connected() is False+++# ---------------------------------------------------------------------------+# Test 2 — connected when token is provided directly+# ---------------------------------------------------------------------------+++def test_connected_with_token() -> None:+ """is_connected() returns True when a token is passed directly."""+ from openjarvis.connectors.notion import NotionConnector # noqa: PLC0415++ conn = NotionConnector(token="ntn_fake")+ assert conn.is_connected() is True+++# ---------------------------------------------------------------------------+# Test 3 — auth_url references notion.so+# ---------------------------------------------------------------------------+++def test_auth_url(connector) -> None:+ """auth_url() returns a URL pointing users to the Notion integrations page."""+ url = connector.auth_url()+ assert "notion.so" in url+++# ---------------------------------------------------------------------------+# Test 4 — sync yields documents with correct fields (mocked API)+# ---------------------------------------------------------------------------+++@patch("openjarvis.connectors.notion._notion_api_search")+@patch("openjarvis.connectors.notion._notion_api_get_blocks")+def test_sync_yields_documents(+ mock_blocks,+ mock_search,+ connector,+ tmp_path: Path,+) -> None:+ """sync() yields one Document per page with correct metadata."""+ # Write fake credentials so is_connected() returns True+ creds_path = Path(connector._credentials_path)+ creds_path.write_text(+ json.dumps({"token": "ntn_fake"}), encoding="utf-8"+ )++ mock_search.return_value = _SEARCH_RESPONSE+ mock_blocks.return_value = _BLOCKS_RESPONSE++ docs: List[Document] = list(connector.sync())++ assert len(docs) == 2++ # --- Page 1 ---+ doc1 = next(d for d in docs if d.doc_id == "notion:page1")+ assert doc1.source == "notion"+ assert doc1.doc_type == "document"+ assert doc1.title == "Meeting Notes"+ assert doc1.url == "https://notion.so/page1"+ assert "Hello world" in doc1.content+ assert "## Section" in doc1.content++ # --- Page 2 ---+ doc2 = next(d for d in docs if d.doc_id == "notion:page2")+ assert doc2.title == "Project Plan"+ assert doc2.url == "https://notion.so/page2"++ # Verify the API was called correctly+ mock_search.assert_called_once()+ assert mock_blocks.call_count == 2+++# ---------------------------------------------------------------------------+# Test 5 — render_blocks_to_markdown handles various block types+# ---------------------------------------------------------------------------+++def test_render_blocks_to_markdown() -> None:+ """_render_blocks_to_markdown converts each block type to the correct markdown."""+ from openjarvis.connectors.notion import _render_blocks_to_markdown # noqa: PLC0415++ blocks = [+ {"type": "paragraph", "paragraph": {"rich_text": [{"plain_text": "Para text"}]}},+ {"type": "heading_1", "heading_1": {"rich_text": [{"plain_text": "H1 Title"}]}},+ {"type": "heading_2", "heading_2": {"rich_text": [{"plain_text": "H2 Title"}]}},+ {"type": "heading_3", "heading_3": {"rich_text": [{"plain_text": "H3 Title"}]}},+ {+ "type": "bulleted_list_item",+ "bulleted_list_item": {"rich_text": [{"plain_text": "Bullet point"}]},+ },+ {+ "type": "numbered_list_item",+ "numbered_list_item": {"rich_text": [{"plain_text": "Numbered item"}]},+ },+ {+ "type": "to_do",+ "to_do": {"rich_text": [{"plain_text": "Unchecked task"}], "checked": False},+ },+ {+ "type": "to_do",+ "to_do": {"rich_text": [{"plain_text": "Checked task"}], "checked": True},+ },+ {+ "type": "code",+ "code": {+ "rich_text": [{"plain_text": "print('hello')"}],+ "language": "python",+ },+ },+ {"type": "quote", "quote": {"rich_text": [{"plain_text": "A wise saying"}]}},+ {"type": "divider"},+ ]++ result = _render_blocks_to_markdown(blocks)++ assert "Para text" in result+ assert "# H1 Title" in result+ assert "## H2 Title" in result+ assert "### H3 Title" in result+ assert "- Bullet point" in result+ assert "1. Numbered item" in result+ assert "- [ ] Unchecked task" in result+ assert "- [x] Checked task" in result+ assert "```" in result+ assert "print('hello')" in result+ assert "> A wise saying" in result+ assert "---" in result+++# ---------------------------------------------------------------------------+# Test 6 — disconnect removes the credentials file+# ---------------------------------------------------------------------------+++def test_disconnect(connector, tmp_path: Path) -> None:+ """disconnect() deletes the credentials file."""+ creds_path = Path(connector._credentials_path)+ creds_path.write_text(+ json.dumps({"token": "ntn_fake"}), encoding="utf-8"+ )+ assert connector.is_connected() is True++ connector.disconnect()++ assert not creds_path.exists()+ assert connector.is_connected() is False+++# ---------------------------------------------------------------------------+# Test 7 — mcp_tools returns the two expected tool specs+# ---------------------------------------------------------------------------+++def test_mcp_tools(connector) -> None:+ """mcp_tools() returns exactly 2 tools with the required names."""+ tools = connector.mcp_tools()+ names = {t.name for t in tools}+ assert len(tools) == 2+ assert "notion_search_pages" in names+ assert "notion_get_page" in names+++# ---------------------------------------------------------------------------+# Test 8 — ConnectorRegistry contains "notion" after import+# ---------------------------------------------------------------------------+++def test_registry() -> None:+ """NotionConnector can be registered and retrieved via ConnectorRegistry."""+ from openjarvis.connectors.notion import NotionConnector # noqa: PLC0415++ ConnectorRegistry.register_value("notion", NotionConnector)+ assert ConnectorRegistry.contains("notion")+ cls = ConnectorRegistry.get("notion")+ assert cls.connector_id == "notion"