Skip to content

Personal AI, On Personal Devices

Python

feat: add Dropbox connector with file listing and download

krypticmouseb776c40507f7
diff --git a/src/openjarvis/connectors/dropbox.py b/src/openjarvis/connectors/dropbox.py
new file mode 100644
index 00000000..d7372db1
--- /dev/null
+++ b/src/openjarvis/connectors/dropbox.py
@@ -0,0 +1,379 @@
+"""Dropbox connector — bulk file sync via the Dropbox API v2.
+
+Uses OAuth Bearer tokens stored locally.
+All network calls are isolated in module-level functions (``_dropbox_api_*``)
+to make them trivially mockable in tests.
+"""
+
+from __future__ import annotations
+
+from datetime import datetime
+from typing import Any, Dict, Iterator, List, Optional
+
+import httpx
+
+from openjarvis.connectors._stubs import BaseConnector, Document, SyncStatus
+from openjarvis.connectors.oauth import delete_tokens, load_tokens, save_tokens
+from openjarvis.core.config import DEFAULT_CONFIG_DIR
+from openjarvis.core.registry import ConnectorRegistry
+from openjarvis.tools._stubs import ToolSpec
+
+# ---------------------------------------------------------------------------
+# Constants
+# ---------------------------------------------------------------------------
+
+_DROPBOX_API_BASE = "https://api.dropboxapi.com/2"
+_DROPBOX_CONTENT_BASE = "https://content.dropboxapi.com/2"
+_DROPBOX_AUTH_URL = "https://www.dropbox.com/oauth2/authorize"
+_DROPBOX_SCOPES = ["files.metadata.read", "files.content.read"]
+_DEFAULT_CREDENTIALS_PATH = str(
+ DEFAULT_CONFIG_DIR / "connectors" / "dropbox.json"
+)
+
+# File extensions whose content can be downloaded and stored as text
+_TEXT_EXTENSIONS = {
+ ".txt",
+ ".md",
+ ".csv",
+ ".py",
+ ".js",
+ ".ts",
+ ".html",
+ ".htm",
+ ".json",
+ ".yaml",
+ ".yml",
+ ".toml",
+ ".ini",
+ ".cfg",
+ ".rst",
+ ".sh",
+ ".log",
+}
+
+# ---------------------------------------------------------------------------
+# Module-level API functions (easy to patch in tests)
+# ---------------------------------------------------------------------------
+
+
+def _dropbox_api_list_folder(
+ token: str,
+ *,
+ path: str = "",
+ cursor: Optional[str] = None,
+) -> Dict[str, Any]:
+ """Call the Dropbox ``files/list_folder`` (or ``/continue``) endpoint.
+
+ Parameters
+ ----------
+ token:
+ OAuth Bearer access token.
+ path:
+ Dropbox path to list. Use ``""`` (empty string) for the root.
+ cursor:
+ Pagination cursor from a previous response; if provided, calls
+ ``files/list_folder/continue`` instead.
+
+ Returns
+ -------
+ dict
+ Raw API response containing ``entries``, ``cursor``, and
+ ``has_more``.
+ """
+ if cursor is not None:
+ resp = httpx.post(
+ f"{_DROPBOX_API_BASE}/files/list_folder/continue",
+ headers={
+ "Authorization": f"Bearer {token}",
+ "Content-Type": "application/json",
+ },
+ json={"cursor": cursor},
+ timeout=30.0,
+ )
+ else:
+ resp = httpx.post(
+ f"{_DROPBOX_API_BASE}/files/list_folder",
+ headers={
+ "Authorization": f"Bearer {token}",
+ "Content-Type": "application/json",
+ },
+ json={"path": path, "recursive": True},
+ timeout=30.0,
+ )
+ resp.raise_for_status()
+ return resp.json()
+
+
+def _dropbox_api_download(token: str, path: str) -> str:
+ """Download a file from Dropbox and return its content as a string.
+
+ Parameters
+ ----------
+ token:
+ OAuth Bearer access token.
+ path:
+ Dropbox path of the file to download.
+
+ Returns
+ -------
+ str
+ File content decoded as UTF-8.
+ """
+ import json as _json
+
+ resp = httpx.post(
+ f"{_DROPBOX_CONTENT_BASE}/files/download",
+ headers={
+ "Authorization": f"Bearer {token}",
+ "Dropbox-API-Arg": _json.dumps({"path": path}),
+ },
+ timeout=60.0,
+ )
+ resp.raise_for_status()
+ return resp.text
+
+
+# ---------------------------------------------------------------------------
+# DropboxConnector
+# ---------------------------------------------------------------------------
+
+
+class DropboxConnector(BaseConnector):
+ """Connector that syncs files from Dropbox via the REST API v2.
+
+ Authentication is handled through Dropbox OAuth 2.0. Tokens are stored
+ locally in a JSON credentials file.
+
+ Parameters
+ ----------
+ credentials_path:
+ Path to the JSON file where OAuth tokens are stored. Defaults to
+ ``~/.openjarvis/connectors/dropbox.json``.
+ """
+
+ connector_id = "dropbox"
+ display_name = "Dropbox"
+ auth_type = "oauth"
+
+ def __init__(self, credentials_path: str = "") -> None:
+ self._credentials_path = credentials_path or _DEFAULT_CREDENTIALS_PATH
+ self._items_synced: int = 0
+ self._items_total: int = 0
+ self._last_sync: Optional[datetime] = None
+ self._last_cursor: Optional[str] = None
+
+ # ------------------------------------------------------------------
+ # BaseConnector interface
+ # ------------------------------------------------------------------
+
+ def is_connected(self) -> bool:
+ """Return ``True`` if a credentials file with a valid token exists."""
+ tokens = load_tokens(self._credentials_path)
+ if tokens is None:
+ return False
+ return bool(tokens)
+
+ def disconnect(self) -> None:
+ """Delete the stored credentials file."""
+ delete_tokens(self._credentials_path)
+
+ def auth_url(self) -> str:
+ """Return a Dropbox OAuth consent URL requesting file access scopes."""
+ from urllib.parse import urlencode
+
+ params = {
+ "client_id": "", # placeholder — real client_id from config
+ "response_type": "code",
+ "token_access_type": "offline",
+ "scope": " ".join(_DROPBOX_SCOPES),
+ }
+ return f"{_DROPBOX_AUTH_URL}?{urlencode(params)}"
+
+ def handle_callback(self, code: str) -> None:
+ """Handle the OAuth callback by persisting the authorization code.
+
+ In a full implementation this would exchange the code for tokens.
+ For now the code is saved directly as the token value.
+ """
+ save_tokens(self._credentials_path, {"token": code})
+
+ def sync(
+ self,
+ *,
+ since: Optional[datetime] = None, # noqa: ARG002 — reserved for future use
+ cursor: Optional[str] = None,
+ ) -> Iterator[Document]:
+ """Yield :class:`Document` objects for Dropbox files.
+
+ Lists all files recursively. Text-extractable files (.txt, .md,
+ .csv, .py, etc.) are downloaded and stored with their content.
+ Other files are stored as metadata-only documents.
+
+ Parameters
+ ----------
+ since:
+ Not yet used (reserved for future incremental sync).
+ cursor:
+ Dropbox list_folder cursor from a previous sync to resume
+ pagination.
+ """
+ tokens = load_tokens(self._credentials_path)
+ if not tokens:
+ return
+
+ token: str = tokens.get("token", tokens.get("access_token", ""))
+ if not token:
+ return
+
+ list_cursor: Optional[str] = cursor
+ synced = 0
+ entries_seen: int = 0
+
+ while True:
+ list_resp = _dropbox_api_list_folder(
+ token, path="", cursor=list_cursor
+ )
+ entries: List[Dict[str, Any]] = list_resp.get("entries", [])
+ entries_seen += len(entries)
+
+ for entry in entries:
+ tag: str = entry.get(".tag", "")
+ if tag != "file":
+ # Skip folders / deleted entries
+ continue
+
+ name: str = entry.get("name", "")
+ path_lower: str = entry.get("path_lower", "")
+ server_modified: str = entry.get("server_modified", "")
+ size: int = entry.get("size", 0)
+
+ # Parse timestamp
+ try:
+ timestamp = datetime.strptime(
+ server_modified, "%Y-%m-%dT%H:%M:%SZ"
+ ).replace(tzinfo=__import__("datetime").timezone.utc)
+ except (ValueError, AttributeError):
+ timestamp = datetime.now()
+
+ # Determine whether the file is text-extractable
+ suffix = "." + name.rsplit(".", 1)[-1].lower() if "." in name else ""
+ if suffix in _TEXT_EXTENSIONS:
+ try:
+ content = _dropbox_api_download(token, path_lower)
+ except Exception: # noqa: BLE001
+ content = f"[File: {name}]"
+ else:
+ content = f"[File: {name}]"
+
+ doc = Document(
+ doc_id=f"dropbox:{path_lower}",
+ source="dropbox",
+ doc_type="document",
+ content=content,
+ title=name,
+ timestamp=timestamp,
+ metadata={
+ "path": path_lower,
+ "size": size,
+ },
+ )
+ synced += 1
+ yield doc
+
+ has_more: bool = list_resp.get("has_more", False)
+ next_cursor: Optional[str] = list_resp.get("cursor")
+ if not has_more:
+ self._last_cursor = next_cursor
+ break
+ list_cursor = next_cursor
+
+ self._items_synced = synced
+ self._items_total = entries_seen
+ self._last_sync = datetime.now()
+
+ def sync_status(self) -> SyncStatus:
+ """Return sync progress from the most recent :meth:`sync` call."""
+ return SyncStatus(
+ state="idle",
+ items_synced=self._items_synced,
+ items_total=self._items_total,
+ last_sync=self._last_sync,
+ cursor=self._last_cursor,
+ )
+
+ # ------------------------------------------------------------------
+ # MCP tools
+ # ------------------------------------------------------------------
+
+ def mcp_tools(self) -> List[ToolSpec]:
+ """Expose three MCP tool specs for real-time Dropbox queries."""
+ return [
+ ToolSpec(
+ name="dropbox_search_files",
+ description=(
+ "Search Dropbox files by filename or content keyword. "
+ "Returns matching files with path and metadata."
+ ),
+ parameters={
+ "type": "object",
+ "properties": {
+ "query": {
+ "type": "string",
+ "description": "Search query string",
+ },
+ "max_results": {
+ "type": "integer",
+ "description": "Maximum number of files to return",
+ "default": 20,
+ },
+ },
+ "required": ["query"],
+ },
+ category="productivity",
+ ),
+ ToolSpec(
+ name="dropbox_get_file",
+ description=(
+ "Retrieve the full text content of a Dropbox file by its path."
+ ),
+ parameters={
+ "type": "object",
+ "properties": {
+ "path": {
+ "type": "string",
+ "description": "Dropbox file path (e.g. '/notes.md')",
+ },
+ },
+ "required": ["path"],
+ },
+ category="productivity",
+ ),
+ ToolSpec(
+ name="dropbox_list_recent",
+ description=(
+ "List recently modified Dropbox files, "
+ "optionally filtered by file extension."
+ ),
+ parameters={
+ "type": "object",
+ "properties": {
+ "extension": {
+ "type": "string",
+ "description": (
+ "Filter by file extension (e.g. '.md', '.txt'). "
+ "Leave empty to list all types."
+ ),
+ "default": "",
+ },
+ "max_results": {
+ "type": "integer",
+ "description": "Maximum number of files to return",
+ "default": 20,
+ },
+ },
+ "required": [],
+ },
+ category="productivity",
+ ),
+ ]
diff --git a/tests/connectors/test_dropbox.py b/tests/connectors/test_dropbox.py
new file mode 100644
index 00000000..5c8fc636
--- /dev/null
+++ b/tests/connectors/test_dropbox.py
@@ -0,0 +1,173 @@
+"""Tests for DropboxConnector — OAuth-authenticated Dropbox sync connector.
+
+All Dropbox API calls are mocked; no network access is required.
+"""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+from typing import List
+from unittest.mock import patch
+
+import pytest
+
+from openjarvis.connectors._stubs import Document
+from openjarvis.core.registry import ConnectorRegistry
+
+# ---------------------------------------------------------------------------
+# Helpers — fake API payloads
+# ---------------------------------------------------------------------------
+
+_LIST_RESPONSE = {
+ "entries": [
+ {
+ "name": "notes.md",
+ "path_lower": "/notes.md",
+ ".tag": "file",
+ "server_modified": "2024-03-15T10:00:00Z",
+ "size": 1024,
+ },
+ {
+ "name": "report.txt",
+ "path_lower": "/docs/report.txt",
+ ".tag": "file",
+ "server_modified": "2024-03-16T11:00:00Z",
+ "size": 2048,
+ },
+ ],
+ "has_more": False,
+ "cursor": "cursor-abc123",
+}
+
+_DOWNLOAD_RESPONSE = "# My Notes\n\nSome content here."
+
+
+# ---------------------------------------------------------------------------
+# Fixtures
+# ---------------------------------------------------------------------------
+
+
+def connector(tmp_path: Path):
+ """DropboxConnector pointing at a tmp credentials path (no file yet)."""
+ from openjarvis.connectors.dropbox import DropboxConnector # noqa: PLC0415
+
+ creds_path = str(tmp_path / "dropbox.json")
+ return DropboxConnector(credentials_path=creds_path)
+
+
+# ---------------------------------------------------------------------------
+# Test 1 — not connected without a credentials file
+# ---------------------------------------------------------------------------
+
+
+def test_not_connected(connector) -> None:
+ """is_connected() returns False when no credentials file exists."""
+ assert connector.is_connected() is False
+
+
+# ---------------------------------------------------------------------------
+# Test 2 — auth_url contains dropbox
+# ---------------------------------------------------------------------------
+
+
+def test_auth_url(connector) -> None:
+ """auth_url() returns a Dropbox OAuth URL."""
+ url = connector.auth_url()
+ assert isinstance(url, str)
+ assert "dropbox" in url.lower()
+
+
+# ---------------------------------------------------------------------------
+# Test 3 — sync yields documents with correct fields (mocked API)
+# ---------------------------------------------------------------------------
+
+
+@patch("openjarvis.connectors.dropbox._dropbox_api_list_folder")
+@patch("openjarvis.connectors.dropbox._dropbox_api_download")
+def test_sync_yields_documents(
+ mock_download,
+ mock_list,
+ connector,
+ tmp_path: Path,
+) -> None:
+ """sync() yields one Document per file with correct metadata."""
+ # Set up fake credentials so is_connected() returns True
+ creds_path = Path(connector._credentials_path)
+ creds_path.write_text(
+ json.dumps({"token": "fake-access-token"}), encoding="utf-8"
+ )
+
+ # Configure mocks
+ mock_list.return_value = _LIST_RESPONSE
+ mock_download.return_value = _DOWNLOAD_RESPONSE
+
+ docs: List[Document] = list(connector.sync())
+
+ assert len(docs) == 2
+
+ # --- Document 1 (notes.md) ---
+ doc1 = next(d for d in docs if d.doc_id == "dropbox:/notes.md")
+ assert doc1.source == "dropbox"
+ assert doc1.doc_type == "document"
+ assert doc1.title == "notes.md"
+ assert doc1.content == _DOWNLOAD_RESPONSE
+
+ # --- Document 2 (report.txt) ---
+ doc2 = next(d for d in docs if d.doc_id == "dropbox:/docs/report.txt")
+ assert doc2.title == "report.txt"
+ assert doc2.content == _DOWNLOAD_RESPONSE
+
+ # Verify the API was called correctly
+ mock_list.assert_called_once()
+ assert mock_download.call_count == 2
+
+
+# ---------------------------------------------------------------------------
+# Test 4 — disconnect removes the credentials file
+# ---------------------------------------------------------------------------
+
+
+def test_disconnect(connector, tmp_path: Path) -> None:
+ """disconnect() deletes the credentials file."""
+ creds_path = Path(connector._credentials_path)
+ creds_path.write_text(
+ json.dumps({"token": "fake-access-token"}), encoding="utf-8"
+ )
+ assert connector.is_connected() is True
+
+ connector.disconnect()
+
+ assert not creds_path.exists()
+ assert connector.is_connected() is False
+
+
+# ---------------------------------------------------------------------------
+# Test 5 — mcp_tools returns the three expected tool specs
+# ---------------------------------------------------------------------------
+
+
+def test_mcp_tools(connector) -> None:
+ """mcp_tools() returns exactly 3 tools with the required names."""
+ tools = connector.mcp_tools()
+ names = {t.name for t in tools}
+ assert len(tools) == 3
+ assert "dropbox_search_files" in names
+ assert "dropbox_get_file" in names
+ assert "dropbox_list_recent" in names
+
+
+# ---------------------------------------------------------------------------
+# Test 6 — ConnectorRegistry contains "dropbox" after import
+# ---------------------------------------------------------------------------
+
+
+def test_registry() -> None:
+ """DropboxConnector can be registered and retrieved via ConnectorRegistry."""
+ from openjarvis.connectors.dropbox import DropboxConnector # noqa: PLC0415
+
+ ConnectorRegistry.register_value("dropbox", DropboxConnector)
+ assert ConnectorRegistry.contains("dropbox")
+ cls = ConnectorRegistry.get("dropbox")
+ assert cls.connector_id == "dropbox"