Personal AI, On Personal Devices
Python
feat: add Dropbox connector with file listing and download
krypticmouseb776c40507f7
diff --git a/src/openjarvis/connectors/dropbox.py b/src/openjarvis/connectors/dropbox.pynew file mode 100644index 00000000..d7372db1--- /dev/null+++ b/src/openjarvis/connectors/dropbox.py@@ -0,0 +1,379 @@+"""Dropbox connector — bulk file sync via the Dropbox API v2.++Uses OAuth Bearer tokens stored locally.+All network calls are isolated in module-level functions (``_dropbox_api_*``)+to make them trivially mockable in tests.+"""++from __future__ import annotations++from datetime import datetime+from typing import Any, Dict, Iterator, List, Optional++import httpx++from openjarvis.connectors._stubs import BaseConnector, Document, SyncStatus+from openjarvis.connectors.oauth import delete_tokens, load_tokens, save_tokens+from openjarvis.core.config import DEFAULT_CONFIG_DIR+from openjarvis.core.registry import ConnectorRegistry+from openjarvis.tools._stubs import ToolSpec++# ---------------------------------------------------------------------------+# Constants+# ---------------------------------------------------------------------------++_DROPBOX_API_BASE = "https://api.dropboxapi.com/2"+_DROPBOX_CONTENT_BASE = "https://content.dropboxapi.com/2"+_DROPBOX_AUTH_URL = "https://www.dropbox.com/oauth2/authorize"+_DROPBOX_SCOPES = ["files.metadata.read", "files.content.read"]+_DEFAULT_CREDENTIALS_PATH = str(+ DEFAULT_CONFIG_DIR / "connectors" / "dropbox.json"+)++# File extensions whose content can be downloaded and stored as text+_TEXT_EXTENSIONS = {+ ".txt",+ ".md",+ ".csv",+ ".py",+ ".js",+ ".ts",+ ".html",+ ".htm",+ ".json",+ ".yaml",+ ".yml",+ ".toml",+ ".ini",+ ".cfg",+ ".rst",+ ".sh",+ ".log",+}++# ---------------------------------------------------------------------------+# Module-level API functions (easy to patch in tests)+# ---------------------------------------------------------------------------+++def _dropbox_api_list_folder(+ token: str,+ *,+ path: str = "",+ cursor: Optional[str] = None,+) -> Dict[str, Any]:+ """Call the Dropbox ``files/list_folder`` (or ``/continue``) endpoint.++ Parameters+ ----------+ token:+ OAuth Bearer access token.+ path:+ Dropbox path to list. Use ``""`` (empty string) for the root.+ cursor:+ Pagination cursor from a previous response; if provided, calls+ ``files/list_folder/continue`` instead.++ Returns+ -------+ dict+ Raw API response containing ``entries``, ``cursor``, and+ ``has_more``.+ """+ if cursor is not None:+ resp = httpx.post(+ f"{_DROPBOX_API_BASE}/files/list_folder/continue",+ headers={+ "Authorization": f"Bearer {token}",+ "Content-Type": "application/json",+ },+ json={"cursor": cursor},+ timeout=30.0,+ )+ else:+ resp = httpx.post(+ f"{_DROPBOX_API_BASE}/files/list_folder",+ headers={+ "Authorization": f"Bearer {token}",+ "Content-Type": "application/json",+ },+ json={"path": path, "recursive": True},+ timeout=30.0,+ )+ resp.raise_for_status()+ return resp.json()+++def _dropbox_api_download(token: str, path: str) -> str:+ """Download a file from Dropbox and return its content as a string.++ Parameters+ ----------+ token:+ OAuth Bearer access token.+ path:+ Dropbox path of the file to download.++ Returns+ -------+ str+ File content decoded as UTF-8.+ """+ import json as _json++ resp = httpx.post(+ f"{_DROPBOX_CONTENT_BASE}/files/download",+ headers={+ "Authorization": f"Bearer {token}",+ "Dropbox-API-Arg": _json.dumps({"path": path}),+ },+ timeout=60.0,+ )+ resp.raise_for_status()+ return resp.text+++# ---------------------------------------------------------------------------+# DropboxConnector+# ---------------------------------------------------------------------------++[email protected]("dropbox")+class DropboxConnector(BaseConnector):+ """Connector that syncs files from Dropbox via the REST API v2.++ Authentication is handled through Dropbox OAuth 2.0. Tokens are stored+ locally in a JSON credentials file.++ Parameters+ ----------+ credentials_path:+ Path to the JSON file where OAuth tokens are stored. Defaults to+ ``~/.openjarvis/connectors/dropbox.json``.+ """++ connector_id = "dropbox"+ display_name = "Dropbox"+ auth_type = "oauth"++ def __init__(self, credentials_path: str = "") -> None:+ self._credentials_path = credentials_path or _DEFAULT_CREDENTIALS_PATH+ self._items_synced: int = 0+ self._items_total: int = 0+ self._last_sync: Optional[datetime] = None+ self._last_cursor: Optional[str] = None++ # ------------------------------------------------------------------+ # BaseConnector interface+ # ------------------------------------------------------------------++ def is_connected(self) -> bool:+ """Return ``True`` if a credentials file with a valid token exists."""+ tokens = load_tokens(self._credentials_path)+ if tokens is None:+ return False+ return bool(tokens)++ def disconnect(self) -> None:+ """Delete the stored credentials file."""+ delete_tokens(self._credentials_path)++ def auth_url(self) -> str:+ """Return a Dropbox OAuth consent URL requesting file access scopes."""+ from urllib.parse import urlencode++ params = {+ "client_id": "", # placeholder — real client_id from config+ "response_type": "code",+ "token_access_type": "offline",+ "scope": " ".join(_DROPBOX_SCOPES),+ }+ return f"{_DROPBOX_AUTH_URL}?{urlencode(params)}"++ def handle_callback(self, code: str) -> None:+ """Handle the OAuth callback by persisting the authorization code.++ In a full implementation this would exchange the code for tokens.+ For now the code is saved directly as the token value.+ """+ save_tokens(self._credentials_path, {"token": code})++ def sync(+ self,+ *,+ since: Optional[datetime] = None, # noqa: ARG002 — reserved for future use+ cursor: Optional[str] = None,+ ) -> Iterator[Document]:+ """Yield :class:`Document` objects for Dropbox files.++ Lists all files recursively. Text-extractable files (.txt, .md,+ .csv, .py, etc.) are downloaded and stored with their content.+ Other files are stored as metadata-only documents.++ Parameters+ ----------+ since:+ Not yet used (reserved for future incremental sync).+ cursor:+ Dropbox list_folder cursor from a previous sync to resume+ pagination.+ """+ tokens = load_tokens(self._credentials_path)+ if not tokens:+ return++ token: str = tokens.get("token", tokens.get("access_token", ""))+ if not token:+ return++ list_cursor: Optional[str] = cursor+ synced = 0+ entries_seen: int = 0++ while True:+ list_resp = _dropbox_api_list_folder(+ token, path="", cursor=list_cursor+ )+ entries: List[Dict[str, Any]] = list_resp.get("entries", [])+ entries_seen += len(entries)++ for entry in entries:+ tag: str = entry.get(".tag", "")+ if tag != "file":+ # Skip folders / deleted entries+ continue++ name: str = entry.get("name", "")+ path_lower: str = entry.get("path_lower", "")+ server_modified: str = entry.get("server_modified", "")+ size: int = entry.get("size", 0)++ # Parse timestamp+ try:+ timestamp = datetime.strptime(+ server_modified, "%Y-%m-%dT%H:%M:%SZ"+ ).replace(tzinfo=__import__("datetime").timezone.utc)+ except (ValueError, AttributeError):+ timestamp = datetime.now()++ # Determine whether the file is text-extractable+ suffix = "." + name.rsplit(".", 1)[-1].lower() if "." in name else ""+ if suffix in _TEXT_EXTENSIONS:+ try:+ content = _dropbox_api_download(token, path_lower)+ except Exception: # noqa: BLE001+ content = f"[File: {name}]"+ else:+ content = f"[File: {name}]"++ doc = Document(+ doc_id=f"dropbox:{path_lower}",+ source="dropbox",+ doc_type="document",+ content=content,+ title=name,+ timestamp=timestamp,+ metadata={+ "path": path_lower,+ "size": size,+ },+ )+ synced += 1+ yield doc++ has_more: bool = list_resp.get("has_more", False)+ next_cursor: Optional[str] = list_resp.get("cursor")+ if not has_more:+ self._last_cursor = next_cursor+ break+ list_cursor = next_cursor++ self._items_synced = synced+ self._items_total = entries_seen+ self._last_sync = datetime.now()++ def sync_status(self) -> SyncStatus:+ """Return sync progress from the most recent :meth:`sync` call."""+ return SyncStatus(+ state="idle",+ items_synced=self._items_synced,+ items_total=self._items_total,+ last_sync=self._last_sync,+ cursor=self._last_cursor,+ )++ # ------------------------------------------------------------------+ # MCP tools+ # ------------------------------------------------------------------++ def mcp_tools(self) -> List[ToolSpec]:+ """Expose three MCP tool specs for real-time Dropbox queries."""+ return [+ ToolSpec(+ name="dropbox_search_files",+ description=(+ "Search Dropbox files by filename or content keyword. "+ "Returns matching files with path and metadata."+ ),+ parameters={+ "type": "object",+ "properties": {+ "query": {+ "type": "string",+ "description": "Search query string",+ },+ "max_results": {+ "type": "integer",+ "description": "Maximum number of files to return",+ "default": 20,+ },+ },+ "required": ["query"],+ },+ category="productivity",+ ),+ ToolSpec(+ name="dropbox_get_file",+ description=(+ "Retrieve the full text content of a Dropbox file by its path."+ ),+ parameters={+ "type": "object",+ "properties": {+ "path": {+ "type": "string",+ "description": "Dropbox file path (e.g. '/notes.md')",+ },+ },+ "required": ["path"],+ },+ category="productivity",+ ),+ ToolSpec(+ name="dropbox_list_recent",+ description=(+ "List recently modified Dropbox files, "+ "optionally filtered by file extension."+ ),+ parameters={+ "type": "object",+ "properties": {+ "extension": {+ "type": "string",+ "description": (+ "Filter by file extension (e.g. '.md', '.txt'). "+ "Leave empty to list all types."+ ),+ "default": "",+ },+ "max_results": {+ "type": "integer",+ "description": "Maximum number of files to return",+ "default": 20,+ },+ },+ "required": [],+ },+ category="productivity",+ ),+ ]diff --git a/tests/connectors/test_dropbox.py b/tests/connectors/test_dropbox.pynew file mode 100644index 00000000..5c8fc636--- /dev/null+++ b/tests/connectors/test_dropbox.py@@ -0,0 +1,173 @@+"""Tests for DropboxConnector — OAuth-authenticated Dropbox sync connector.++All Dropbox API calls are mocked; no network access is required.+"""++from __future__ import annotations++import json+from pathlib import Path+from typing import List+from unittest.mock import patch++import pytest++from openjarvis.connectors._stubs import Document+from openjarvis.core.registry import ConnectorRegistry++# ---------------------------------------------------------------------------+# Helpers — fake API payloads+# ---------------------------------------------------------------------------++_LIST_RESPONSE = {+ "entries": [+ {+ "name": "notes.md",+ "path_lower": "/notes.md",+ ".tag": "file",+ "server_modified": "2024-03-15T10:00:00Z",+ "size": 1024,+ },+ {+ "name": "report.txt",+ "path_lower": "/docs/report.txt",+ ".tag": "file",+ "server_modified": "2024-03-16T11:00:00Z",+ "size": 2048,+ },+ ],+ "has_more": False,+ "cursor": "cursor-abc123",+}++_DOWNLOAD_RESPONSE = "# My Notes\n\nSome content here."+++# ---------------------------------------------------------------------------+# Fixtures+# ---------------------------------------------------------------------------+++def connector(tmp_path: Path):+ """DropboxConnector pointing at a tmp credentials path (no file yet)."""+ from openjarvis.connectors.dropbox import DropboxConnector # noqa: PLC0415++ creds_path = str(tmp_path / "dropbox.json")+ return DropboxConnector(credentials_path=creds_path)+++# ---------------------------------------------------------------------------+# Test 1 — not connected without a credentials file+# ---------------------------------------------------------------------------+++def test_not_connected(connector) -> None:+ """is_connected() returns False when no credentials file exists."""+ assert connector.is_connected() is False+++# ---------------------------------------------------------------------------+# Test 2 — auth_url contains dropbox+# ---------------------------------------------------------------------------+++def test_auth_url(connector) -> None:+ """auth_url() returns a Dropbox OAuth URL."""+ url = connector.auth_url()+ assert isinstance(url, str)+ assert "dropbox" in url.lower()+++# ---------------------------------------------------------------------------+# Test 3 — sync yields documents with correct fields (mocked API)+# ---------------------------------------------------------------------------+++@patch("openjarvis.connectors.dropbox._dropbox_api_list_folder")+@patch("openjarvis.connectors.dropbox._dropbox_api_download")+def test_sync_yields_documents(+ mock_download,+ mock_list,+ connector,+ tmp_path: Path,+) -> None:+ """sync() yields one Document per file with correct metadata."""+ # Set up fake credentials so is_connected() returns True+ creds_path = Path(connector._credentials_path)+ creds_path.write_text(+ json.dumps({"token": "fake-access-token"}), encoding="utf-8"+ )++ # Configure mocks+ mock_list.return_value = _LIST_RESPONSE+ mock_download.return_value = _DOWNLOAD_RESPONSE++ docs: List[Document] = list(connector.sync())++ assert len(docs) == 2++ # --- Document 1 (notes.md) ---+ doc1 = next(d for d in docs if d.doc_id == "dropbox:/notes.md")+ assert doc1.source == "dropbox"+ assert doc1.doc_type == "document"+ assert doc1.title == "notes.md"+ assert doc1.content == _DOWNLOAD_RESPONSE++ # --- Document 2 (report.txt) ---+ doc2 = next(d for d in docs if d.doc_id == "dropbox:/docs/report.txt")+ assert doc2.title == "report.txt"+ assert doc2.content == _DOWNLOAD_RESPONSE++ # Verify the API was called correctly+ mock_list.assert_called_once()+ assert mock_download.call_count == 2+++# ---------------------------------------------------------------------------+# Test 4 — disconnect removes the credentials file+# ---------------------------------------------------------------------------+++def test_disconnect(connector, tmp_path: Path) -> None:+ """disconnect() deletes the credentials file."""+ creds_path = Path(connector._credentials_path)+ creds_path.write_text(+ json.dumps({"token": "fake-access-token"}), encoding="utf-8"+ )+ assert connector.is_connected() is True++ connector.disconnect()++ assert not creds_path.exists()+ assert connector.is_connected() is False+++# ---------------------------------------------------------------------------+# Test 5 — mcp_tools returns the three expected tool specs+# ---------------------------------------------------------------------------+++def test_mcp_tools(connector) -> None:+ """mcp_tools() returns exactly 3 tools with the required names."""+ tools = connector.mcp_tools()+ names = {t.name for t in tools}+ assert len(tools) == 3+ assert "dropbox_search_files" in names+ assert "dropbox_get_file" in names+ assert "dropbox_list_recent" in names+++# ---------------------------------------------------------------------------+# Test 6 — ConnectorRegistry contains "dropbox" after import+# ---------------------------------------------------------------------------+++def test_registry() -> None:+ """DropboxConnector can be registered and retrieved via ConnectorRegistry."""+ from openjarvis.connectors.dropbox import DropboxConnector # noqa: PLC0415++ ConnectorRegistry.register_value("dropbox", DropboxConnector)+ assert ConnectorRegistry.contains("dropbox")+ cls = ConnectorRegistry.get("dropbox")+ assert cls.connector_id == "dropbox"