Skip to content

Personal AI, On Personal Devices

Python

feat: add WhatsApp chat export connector

krypticmouse526cd4d3fb04
diff --git a/src/openjarvis/connectors/__init__.py b/src/openjarvis/connectors/__init__.py
index e44b7e02..d8142600 100644
--- a/src/openjarvis/connectors/__init__.py
+++ b/src/openjarvis/connectors/__init__.py
@@ -52,3 +52,8 @@ try:
import openjarvis.connectors.dropbox # noqa: F401
except ImportError:
pass # httpx may not be installed
+
+try:
+ import openjarvis.connectors.whatsapp # noqa: F401
+except ImportError:
+ pass
diff --git a/src/openjarvis/connectors/whatsapp.py b/src/openjarvis/connectors/whatsapp.py
new file mode 100644
index 00000000..0494de08
--- /dev/null
+++ b/src/openjarvis/connectors/whatsapp.py
@@ -0,0 +1,298 @@
+"""WhatsApp chat export connector.
+
+Reads WhatsApp chat history from exported .txt files. WhatsApp allows
+exporting individual chats via *Chat → More → Export Chat* on iOS/Android,
+producing plain-text files in a well-known format.
+
+Export line format::
+
+ 1/15/24, 10:30 AM - Alice: Hey, how are you?
+ 1/15/24, 10:31 AM - Bob: Good thanks! Working on the project.
+
+Each .txt file in the configured directory is parsed and yielded as a
+single :class:`~openjarvis.connectors._stubs.Document` with ``doc_type``
+``"message"`` and the message lines joined as content.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import logging
+import re
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Iterator, List, Optional
+
+from openjarvis.connectors._stubs import BaseConnector, Document, SyncStatus
+from openjarvis.core.registry import ConnectorRegistry
+from openjarvis.tools._stubs import ToolSpec
+
+logger = logging.getLogger(__name__)
+
+# ---------------------------------------------------------------------------
+# Export line regex
+# ---------------------------------------------------------------------------
+
+# Matches lines of the form:
+# 1/15/24, 10:30 AM - Sender: Message text
+# 12/31/2024, 9:05 PM - Alice: Some message
+_LINE_RE = re.compile(
+ r"^(\d{1,2}/\d{1,2}/\d{2,4},\s+\d{1,2}:\d{2}\s*[APap][Mm])\s*-\s*(.+?):\s*(.+)$"
+)
+
+# Date formats tried in order. WhatsApp varies by locale/OS version.
+_DATE_FORMATS = [
+ "%m/%d/%y, %I:%M %p", # 1/15/24, 10:30 AM
+ "%m/%d/%Y, %I:%M %p", # 1/15/2024, 10:30 AM
+ "%d/%m/%y, %I:%M %p", # 15/1/24, 10:30 AM (some locales)
+ "%d/%m/%Y, %I:%M %p", # 15/1/2024, 10:30 AM
+]
+
+
+def _parse_timestamp(ts_str: str) -> Optional[datetime]:
+ """Try to parse a WhatsApp timestamp string into a UTC-aware datetime."""
+ # Normalise whitespace in AM/PM (the regex allows optional space)
+ ts_str = ts_str.strip()
+ for fmt in _DATE_FORMATS:
+ try:
+ dt = datetime.strptime(ts_str, fmt)
+ return dt.replace(tzinfo=timezone.utc)
+ except ValueError:
+ continue
+ logger.debug("Could not parse WhatsApp timestamp: %r", ts_str)
+ return None
+
+
+# ---------------------------------------------------------------------------
+# WhatsAppConnector
+# ---------------------------------------------------------------------------
+
+
[email protected]("whatsapp")
+class WhatsAppConnector(BaseConnector):
+ """Read WhatsApp message history from exported .txt files.
+
+ WhatsApp allows exporting individual chats as plain-text files via
+ *Chat → More → Export Chat*. Point this connector at the directory
+ containing those files (one file per chat) and it will parse each file
+ into a :class:`~openjarvis.connectors._stubs.Document`.
+
+ Parameters
+ ----------
+ export_path:
+ Path to the directory containing WhatsApp export .txt files.
+ If empty, no documents are yielded.
+ """
+
+ connector_id = "whatsapp"
+ display_name = "WhatsApp"
+ auth_type = "filesystem"
+
+ def __init__(self, export_path: str = "") -> None:
+ self._export_path = Path(export_path) if export_path else Path("")
+ self._items_synced: int = 0
+ self._items_total: int = 0
+ self._last_sync: Optional[datetime] = None
+
+ # ------------------------------------------------------------------
+ # BaseConnector interface
+ # ------------------------------------------------------------------
+
+ def is_connected(self) -> bool:
+ """Return ``True`` if the export directory exists and contains .txt files."""
+ if not self._export_path or not self._export_path.is_dir():
+ return False
+ return any(self._export_path.glob("*.txt"))
+
+ def disconnect(self) -> None:
+ """No-op: filesystem connector has nothing to disconnect."""
+
+ def sync(
+ self,
+ *,
+ since: Optional[datetime] = None,
+ cursor: Optional[str] = None, # noqa: ARG002
+ ) -> Iterator[Document]:
+ """Parse WhatsApp export files and yield one Document per chat file.
+
+ Parameters
+ ----------
+ since:
+ If provided, skip files (chats) whose most recent message is
+ before this datetime. A file is included if *any* of its
+ messages are on or after *since*.
+ cursor:
+ Not used (included for API compatibility).
+
+ Yields
+ ------
+ Document
+ One document per .txt export file. The document content is the
+ full parsed text of the chat with timestamp and sender prefixes.
+ Participants are extracted from the set of senders found.
+ """
+ if not self._export_path or not self._export_path.is_dir():
+ return
+
+ txt_files = sorted(self._export_path.glob("*.txt"))
+ self._items_total = len(txt_files)
+ synced = 0
+
+ since_utc: Optional[datetime] = None
+ if since is not None:
+ if since.tzinfo is None:
+ since_utc = since.replace(tzinfo=timezone.utc)
+ else:
+ since_utc = since
+
+ for txt_file in txt_files:
+ doc = self._parse_export_file(txt_file, since=since_utc)
+ if doc is not None:
+ synced += 1
+ yield doc
+
+ self._items_synced = synced
+ self._last_sync = datetime.now(tz=timezone.utc)
+
+ def sync_status(self) -> SyncStatus:
+ """Return sync progress from the most recent :meth:`sync` call."""
+ return SyncStatus(
+ state="idle",
+ items_synced=self._items_synced,
+ items_total=self._items_total,
+ last_sync=self._last_sync,
+ )
+
+ # ------------------------------------------------------------------
+ # MCP tools
+ # ------------------------------------------------------------------
+
+ def mcp_tools(self) -> List[ToolSpec]:
+ """Expose two MCP tool specs for real-time WhatsApp queries."""
+ return [
+ ToolSpec(
+ name="whatsapp_search_messages",
+ description=(
+ "Search WhatsApp messages by keyword or sender name. "
+ "Returns matching messages with sender and timestamp."
+ ),
+ parameters={
+ "type": "object",
+ "properties": {
+ "query": {
+ "type": "string",
+ "description": "Search query string",
+ },
+ "max_results": {
+ "type": "integer",
+ "description": "Maximum number of messages to return",
+ "default": 20,
+ },
+ },
+ "required": ["query"],
+ },
+ category="knowledge",
+ ),
+ ToolSpec(
+ name="whatsapp_get_chat",
+ description=(
+ "Retrieve the full message history for a specific WhatsApp chat "
+ "by chat name (the export filename without the .txt extension)."
+ ),
+ parameters={
+ "type": "object",
+ "properties": {
+ "chat_name": {
+ "type": "string",
+ "description": (
+ "Name of the chat (matches the .txt filename, "
+ "e.g. 'WhatsApp Chat with Alice')"
+ ),
+ },
+ "max_results": {
+ "type": "integer",
+ "description": "Maximum number of messages to return",
+ "default": 50,
+ },
+ },
+ "required": ["chat_name"],
+ },
+ category="knowledge",
+ ),
+ ]
+
+ # ------------------------------------------------------------------
+ # Internal helpers
+ # ------------------------------------------------------------------
+
+ def _parse_export_file(
+ self, path: Path, *, since: Optional[datetime] = None
+ ) -> Optional[Document]:
+ """Parse a single WhatsApp export .txt file.
+
+ Returns a :class:`Document` if the file contains parseable messages
+ that are on or after *since*, or ``None`` if it should be skipped.
+ """
+ try:
+ raw = path.read_text(encoding="utf-8", errors="replace")
+ except OSError:
+ logger.warning("Could not read WhatsApp export file: %s", path)
+ return None
+
+ chat_name = path.stem
+ lines: List[str] = []
+ participants: set[str] = set()
+ latest_ts: Optional[datetime] = None
+ earliest_ts: Optional[datetime] = None
+
+ for raw_line in raw.splitlines():
+ m = _LINE_RE.match(raw_line.strip())
+ if not m:
+ # System message or continuation line — append to previous
+ if lines:
+ lines[-1] = lines[-1] + " " + raw_line.strip()
+ continue
+
+ ts_str, sender, text = m.group(1), m.group(2), m.group(3)
+ dt = _parse_timestamp(ts_str)
+
+ participants.add(sender)
+
+ if dt is not None:
+ if earliest_ts is None or dt < earliest_ts:
+ earliest_ts = dt
+ if latest_ts is None or dt > latest_ts:
+ latest_ts = dt
+
+ # Format: "[timestamp] Sender: message"
+ ts_label = dt.strftime("%Y-%m-%d %H:%M") if dt else ts_str
+ lines.append(f"[{ts_label}] {sender}: {text}")
+
+ if not lines:
+ return None
+
+ # Apply since filter: skip the whole chat if the latest message
+ # predates the since threshold.
+ if since is not None and latest_ts is not None and latest_ts < since:
+ return None
+
+ content = "\n".join(lines)
+ doc_id = f"whatsapp:{hashlib.sha1(path.name.encode()).hexdigest()[:16]}"
+
+ return Document(
+ doc_id=doc_id,
+ source="whatsapp",
+ doc_type="message",
+ content=content,
+ title=chat_name,
+ participants=sorted(participants),
+ timestamp=earliest_ts or datetime.now(tz=timezone.utc),
+ metadata={
+ "chat_name": chat_name,
+ "export_file": str(path),
+ "message_count": len(lines),
+ },
+ )
+
+
+__all__ = ["WhatsAppConnector"]
diff --git a/tests/connectors/test_whatsapp.py b/tests/connectors/test_whatsapp.py
new file mode 100644
index 00000000..9f4cd85b
--- /dev/null
+++ b/tests/connectors/test_whatsapp.py
@@ -0,0 +1,194 @@
+"""Tests for the WhatsApp chat export connector."""
+
+from __future__ import annotations
+
+from datetime import datetime, timezone
+from pathlib import Path
+
+from openjarvis.connectors.whatsapp import WhatsAppConnector
+from openjarvis.core.registry import ConnectorRegistry
+
+# ---------------------------------------------------------------------------
+# Helpers
+# ---------------------------------------------------------------------------
+
+_CHAT_ALICE = """\
+1/15/24, 10:30 AM - Alice: Hey, how are you?
+1/15/24, 10:31 AM - Bob: Good thanks! Working on the project.
+1/15/24, 10:32 AM - Alice: Great, let's discuss later.
+"""
+
+_CHAT_WORK = """\
+3/20/24, 9:00 AM - Carol: Morning everyone!
+3/20/24, 9:01 AM - Dave: Good morning, ready for the standup?
+3/20/24, 9:02 AM - Carol: Yes, joining in 2 minutes.
+3/20/24, 9:05 AM - Eve: Running a bit late, sorry.
+"""
+
+_CHAT_OLD = """\
+6/10/23, 3:00 PM - Frank: This is an old message.
+6/10/23, 3:01 PM - Grace: Indeed, from last year.
+"""
+
+
+def _write_chat(directory: Path, filename: str, content: str) -> Path:
+ """Write a WhatsApp export .txt file and return its path."""
+ p = directory / filename
+ p.write_text(content, encoding="utf-8")
+ return p
+
+
+# ---------------------------------------------------------------------------
+# Test 1: Parse a single export file → one Document with correct fields
+# ---------------------------------------------------------------------------
+
+
+def test_parse_single_export_file(tmp_path: Path) -> None:
+ """A single .txt file is parsed into one Document with correct metadata."""
+ _write_chat(tmp_path, "WhatsApp Chat with Alice.txt", _CHAT_ALICE)
+
+ conn = WhatsAppConnector(export_path=str(tmp_path))
+ docs = list(conn.sync())
+
+ assert len(docs) == 1
+ doc = docs[0]
+ assert doc.source == "whatsapp"
+ assert doc.doc_type == "message"
+ assert doc.title == "WhatsApp Chat with Alice"
+ # Both participants should be detected
+ assert "Alice" in doc.participants
+ assert "Bob" in doc.participants
+ # Content should contain the messages
+ assert "how are you" in doc.content
+ assert "Working on the project" in doc.content
+
+
+# ---------------------------------------------------------------------------
+# Test 2: Multiple export files → one Document per file
+# ---------------------------------------------------------------------------
+
+
+def test_multiple_export_files(tmp_path: Path) -> None:
+ """Multiple .txt files each produce one Document."""
+ _write_chat(tmp_path, "WhatsApp Chat with Alice.txt", _CHAT_ALICE)
+ _write_chat(tmp_path, "Work Group.txt", _CHAT_WORK)
+
+ conn = WhatsAppConnector(export_path=str(tmp_path))
+ docs = list(conn.sync())
+
+ assert len(docs) == 2
+ titles = {d.title for d in docs}
+ assert "WhatsApp Chat with Alice" in titles
+ assert "Work Group" in titles
+
+ # Work group chat has 4 participants
+ work_doc = next(d for d in docs if d.title == "Work Group")
+ assert set(work_doc.participants) == {"Carol", "Dave", "Eve"}
+
+
+# ---------------------------------------------------------------------------
+# Test 3: since filtering — skip chats whose latest message is before cutoff
+# ---------------------------------------------------------------------------
+
+
+def test_since_filtering_skips_old_chats(tmp_path: Path) -> None:
+ """Chats where the most recent message is before `since` are skipped."""
+ _write_chat(tmp_path, "WhatsApp Chat with Alice.txt", _CHAT_ALICE) # Jan 2024
+ _write_chat(tmp_path, "Old Chat.txt", _CHAT_OLD) # Jun 2023
+
+ # Only include chats with messages after Feb 2024
+ cutoff = datetime(2024, 2, 1, tzinfo=timezone.utc)
+
+ conn = WhatsAppConnector(export_path=str(tmp_path))
+ docs = list(conn.sync(since=cutoff))
+
+ # Old chat (Jun 2023) should be excluded; Alice chat (Jan 2024) also before Feb 2024
+ # Both are before Feb 2024 cutoff
+ assert len(docs) == 0
+
+
+def test_since_filtering_includes_recent_chats(tmp_path: Path) -> None:
+ """Chats with messages on or after `since` are included."""
+ _write_chat(tmp_path, "Work Group.txt", _CHAT_WORK) # Mar 2024
+ _write_chat(tmp_path, "Old Chat.txt", _CHAT_OLD) # Jun 2023
+
+ # Cutoff between old and work chat
+ cutoff = datetime(2024, 1, 1, tzinfo=timezone.utc)
+
+ conn = WhatsAppConnector(export_path=str(tmp_path))
+ docs = list(conn.sync(since=cutoff))
+
+ assert len(docs) == 1
+ assert docs[0].title == "Work Group"
+
+
+# ---------------------------------------------------------------------------
+# Test 4: is_connected returns False for missing / empty directory
+# ---------------------------------------------------------------------------
+
+
+def test_is_connected_missing_directory() -> None:
+ """is_connected() returns False when export_path does not exist."""
+ conn = WhatsAppConnector(export_path="/nonexistent/path/to/exports")
+ assert not conn.is_connected()
+
+
+def test_is_connected_empty_directory(tmp_path: Path) -> None:
+ """is_connected() returns False when export directory has no .txt files."""
+ conn = WhatsAppConnector(export_path=str(tmp_path))
+ assert not conn.is_connected()
+
+
+def test_is_connected_with_txt_files(tmp_path: Path) -> None:
+ """is_connected() returns True when at least one .txt file exists."""
+ _write_chat(tmp_path, "Chat.txt", _CHAT_ALICE)
+ conn = WhatsAppConnector(export_path=str(tmp_path))
+ assert conn.is_connected()
+
+
+# ---------------------------------------------------------------------------
+# Test 5: sync with no export_path → empty
+# ---------------------------------------------------------------------------
+
+
+def test_sync_no_export_path() -> None:
+ """sync() yields nothing when no export_path is set."""
+ conn = WhatsAppConnector()
+ docs = list(conn.sync())
+ assert docs == []
+
+
+# ---------------------------------------------------------------------------
+# Test 6: sync_status reflects items synced
+# ---------------------------------------------------------------------------
+
+
+def test_sync_status_after_sync(tmp_path: Path) -> None:
+ """sync_status() reports items_synced and items_total correctly."""
+ _write_chat(tmp_path, "Alice.txt", _CHAT_ALICE)
+ _write_chat(tmp_path, "Work.txt", _CHAT_WORK)
+
+ conn = WhatsAppConnector(export_path=str(tmp_path))
+ docs = list(conn.sync())
+
+ status = conn.sync_status()
+ assert status.items_total == 2
+ assert status.items_synced == len(docs)
+ assert status.last_sync is not None
+
+
+# ---------------------------------------------------------------------------
+# Test 7: ConnectorRegistry contains "whatsapp"
+# ---------------------------------------------------------------------------
+
+
+def test_registry_registration() -> None:
+ """WhatsAppConnector is registered under 'whatsapp' in ConnectorRegistry."""
+ from openjarvis.connectors.whatsapp import WhatsAppConnector # noqa: PLC0415
+
+ ConnectorRegistry.register_value("whatsapp", WhatsAppConnector)
+ assert ConnectorRegistry.contains("whatsapp")
+ cls = ConnectorRegistry.get("whatsapp")
+ assert cls.connector_id == "whatsapp"
+ assert cls.display_name == "WhatsApp"
+ assert cls.auth_type == "filesystem"