RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
refactor: remove python implemention of whatsapp channel (#20432)
Jin Hai62972b1525b0
diff --git a/api/channels/whatsapp/__init__.py b/api/channels/whatsapp/__init__.pydeleted file mode 100644index 0b15b0677b..0000000000--- a/api/channels/whatsapp/__init__.py+++ /dev/null@@ -1 +0,0 @@-from . import channel # noqa: F401diff --git a/api/channels/whatsapp/channel.py b/api/channels/whatsapp/channel.pydeleted file mode 100644index b3a9966d2f..0000000000--- a/api/channels/whatsapp/channel.py+++ /dev/null@@ -1,383 +0,0 @@-from __future__ import annotations--import asyncio-import json-import logging-import time-from dataclasses import dataclass-from typing import Any, Optional--import aiohttp--from ..core.base import Channel, IncomingMessage, OutgoingMessage-from ..core.registry import register_channel--LOGGER = logging.getLogger(__name__)--WHATSAPP_DEFAULT_TIMEOUT_SECS = 30-WHATSAPP_DEFAULT_SESSION_KEY = "default"-WHATSAPP_MESSAGE_TTL_SECS = 3600-WHATSAPP_GATEWAY_START_RETRY_SECS = 30-WHATSAPP_GATEWAY_RECONNECT_SECS = 3---@dataclass-class WhatsAppAccount:- account_id: str- gateway_base_url: str- gateway_token: str = ""- session_key: str = ""- timeout_secs: int = WHATSAPP_DEFAULT_TIMEOUT_SECS---_live_channels: dict[str, "WhatsAppChannel"] = {}---def _default_gateway_base_url() -> str:- return "http://127.0.0.1:3005"---def get_runtime_snapshot(account_id: str) -> dict[str, Any] | None:- channel = _live_channels.get(account_id)- if channel is None:- return None- return channel.get_status_snapshot()---class WhatsAppChannel(Channel):- channel_id = "whatsapp"-- def __init__(self, account: WhatsAppAccount) -> None:- super().__init__()- self.account = account- self.account_id = account.account_id- self._task: Optional[asyncio.Task] = None- self._stop_event = asyncio.Event()- self._lifecycle_lock = asyncio.Lock()- self._http_session: Optional[aiohttp.ClientSession] = None- self._status: str = "stopped"- self._last_error: str = ""- self._qr_data_url: str = ""- self._qr_updated_at: float = 0.0- self._connected_at: float = 0.0- self._last_snapshot_at: float = 0.0- self._session_id: str = ""- self._event_cursor: int = 0- self._seen_message_ids: dict[str, float] = {}-- def _session_key(self) -> str:- value = str(self.account.session_key or "").strip()- return value or WHATSAPP_DEFAULT_SESSION_KEY-- def _gateway_base_url(self) -> str:- return str(self.account.gateway_base_url or "").strip().rstrip("/")-- def _gateway_url(self, path: str) -> str:- base_url = self._gateway_base_url()- if not base_url:- raise ValueError("WhatsApp gateway_base_url is required")- return f"{base_url}/{path.lstrip('/')}"-- def _events_ws_url(self) -> str:- base_url = self._gateway_base_url()- if not base_url:- raise ValueError("WhatsApp gateway_base_url is required")- if base_url.startswith("http://"):- ws_base = f"ws://{base_url.removeprefix('http://')}"- elif base_url.startswith("https://"):- ws_base = f"wss://{base_url.removeprefix('https://')}"- else:- ws_base = base_url- return f"{ws_base}/whatsapp/{self._session_key()}/events/ws?after={self._event_cursor}"-- def _gateway_headers(self) -> dict[str, str]:- token = str(self.account.gateway_token or "").strip()- if not token:- return {}- if token.lower().startswith("bearer "):- return {"Authorization": token}- return {"Authorization": f"Bearer {token}"}-- async def _ensure_http_session(self) -> aiohttp.ClientSession:- if self._http_session is not None and not self._http_session.closed:- return self._http_session- timeout = aiohttp.ClientTimeout(total=max(int(self.account.timeout_secs), 1))- self._http_session = aiohttp.ClientSession(timeout=timeout)- return self._http_session-- async def _request_json(- self,- method: str,- path: str,- payload: dict[str, Any] | None = None,- ) -> dict[str, Any]:- session = await self._ensure_http_session()- url = self._gateway_url(path)- async with session.request(- method,- url,- json=payload,- headers=self._gateway_headers(),- ) as resp:- text = await resp.text()- if resp.status >= 400:- raise RuntimeError(f"status: {resp.status}, response: {text}")- if not text.strip():- return {}- content_type = resp.headers.get("content-type", "")- if "application/json" not in content_type.lower():- raise RuntimeError(f"unexpected response content-type: {content_type or 'unknown'}, response: {text[:200]}")- try:- return await resp.json()- except Exception as ex:- raise RuntimeError(f"invalid json response: {text[:200]}") from ex-- async def _request_json_with_retry(- self,- method: str,- path: str,- payload: dict[str, Any] | None = None,- ) -> dict[str, Any]:- deadline = time.time() + WHATSAPP_GATEWAY_START_RETRY_SECS- last_error: Exception | None = None- while time.time() < deadline and not self._stop_event.is_set():- try:- return await self._request_json(method, path, payload)- except (aiohttp.ClientError, asyncio.TimeoutError, OSError) as ex:- last_error = ex- await asyncio.sleep(0.5)- if last_error is not None:- raise last_error- return await self._request_json(method, path, payload)-- async def start(self) -> None:- async with self._lifecycle_lock:- if self._task and not self._task.done():- return- self._stop_event.clear()- _live_channels[self.account_id] = self- self._task = asyncio.create_task(self._run(), name=f"whatsapp-{self.account_id}")- LOGGER.info("[whatsapp:%s] starting gateway client", self.account_id)-- async def stop(self) -> None:- async with self._lifecycle_lock:- self._stop_event.set()- task = self._task- if task and not task.done():- task.cancel()- try:- await task- except (asyncio.CancelledError, Exception):- pass- try:- await self._request_json(- "POST",- f"whatsapp/{self._session_key()}/stop",- {"account_id": self.account_id, "session_key": self._session_key()},- )- except Exception:- LOGGER.debug("[whatsapp:%s] gateway stop failed", self.account_id, exc_info=True)-- try:- if self._http_session is not None and not self._http_session.closed:- await self._http_session.close()- finally:- self._http_session = None-- _live_channels.pop(self.account_id, None)- self._task = None- self._status = "stopped"- self._last_error = ""- self._qr_data_url = ""- self._qr_updated_at = 0.0- self._connected_at = 0.0- self._last_snapshot_at = 0.0- self._session_id = ""- self._event_cursor = 0- self._seen_message_ids.clear()-- async def send(self, message: OutgoingMessage) -> None:- if not message.chat_id:- LOGGER.error("[whatsapp:%s] missing chat_id; cannot send", self.account_id)- return- try:- LOGGER.info(- "[whatsapp:%s] sending reply chat_id=%s reply_to=%s text_preview=%r",- self.account_id,- message.chat_id,- message.reply_to_message_id,- message.text[:120],- )- await self._request_json(- "POST",- f"whatsapp/{self._session_key()}/send",- {- "account_id": self.account_id,- "session_key": self._session_key(),- "chat_id": message.chat_id,- "text": message.text,- "reply_to_message_id": message.reply_to_message_id,- },- )- LOGGER.info(- "[whatsapp:%s] message sent chat_id=%s reply_to=%s",- self.account_id,- message.chat_id,- message.reply_to_message_id,- )- except Exception:- LOGGER.error("[whatsapp:%s] send failed", self.account_id, exc_info=True)-- async def _run(self) -> None:- self._status = "connecting"- self._last_error = ""- try:- await self._request_json_with_retry(- "POST",- f"whatsapp/{self._session_key()}/start",- {- "account_id": self.account_id,- "session_key": self._session_key(),- "gateway_base_url": self._gateway_base_url(),- },- )- while not self._stop_event.is_set():- try:- await self._run_events_ws()- except asyncio.CancelledError:- raise- except Exception as ex:- self._status = "error"- self._last_error = str(ex)- LOGGER.error("[whatsapp:%s] gateway event loop error", self.account_id, exc_info=True)- if not self._stop_event.is_set():- await asyncio.sleep(WHATSAPP_GATEWAY_RECONNECT_SECS)- except asyncio.CancelledError:- pass- except Exception as ex:- self._status = "error"- self._last_error = str(ex)- LOGGER.error("[whatsapp:%s] gateway runtime error", self.account_id, exc_info=True)- finally:- if not self._stop_event.is_set() and self._status != "error":- self._status = "disconnected"- if not self._last_error:- self._last_error = "WhatsApp gateway stopped unexpectedly."- self._last_snapshot_at = time.time()-- def _apply_snapshot(self, snapshot: dict[str, Any]) -> None:- self._last_snapshot_at = time.time()- self._status = str(snapshot.get("status") or "connecting")- self._last_error = str(snapshot.get("last_error") or "")- self._qr_data_url = str(snapshot.get("qr_data_url") or "")- self._qr_updated_at = float(snapshot.get("qr_updated_at") or 0.0)- self._connected_at = float(snapshot.get("connected_at") or 0.0)- self._session_id = str(snapshot.get("session_id") or self._session_id or "")- self._event_cursor = max(self._event_cursor, int(snapshot.get("event_cursor") or 0))- if self._status == "connected":- self._last_error = ""-- async def _run_events_ws(self) -> None:- session = await self._ensure_http_session()- url = self._events_ws_url()- async with session.ws_connect(url, headers=self._gateway_headers(), heartbeat=None) as ws:- async for msg in ws:- if msg.type == aiohttp.WSMsgType.TEXT:- await self._handle_ws_payload(msg.data)- elif msg.type == aiohttp.WSMsgType.ERROR:- raise RuntimeError(f"whatsapp events websocket error: {ws.exception()}")- LOGGER.warning("[whatsapp:%s] events websocket closed", self.account_id)-- async def _handle_ws_payload(self, payload: str) -> None:- try:- obj = json.loads(payload)- except json.JSONDecodeError:- LOGGER.warning(- "[whatsapp:%s] ignored invalid websocket payload: %r",- self.account_id,- payload[:200],- )- return-- kind = str(obj.get("type") or "").strip()- data = obj.get("data")- if kind == "snapshot" and isinstance(data, dict):- self._apply_snapshot(data)- return- if kind == "event" and isinstance(data, dict):- await self._handle_event_item(data)- return-- async def _handle_event_item(self, item: dict[str, Any]) -> None:- seq = int(item.get("seq") or 0)- if seq > self._event_cursor:- self._event_cursor = seq- if item.get("kind") != "message":- return- message_id = str(item.get("message_id") or "").strip()- if not message_id:- return- self._prune_seen_message_ids()- if self._is_seen_message(message_id):- return- self._seen_message_ids[message_id] = time.time()- incoming = IncomingMessage(- channel=self.channel_id,- account_id=self.account_id,- chat_id=str(item.get("chat_id") or ""),- chat_type=str(item.get("chat_type") or "p2p"),- message_id=message_id,- sender_id=str(item.get("sender_id") or ""),- text=str(item.get("text") or ""),- raw=item.get("raw"),- )- LOGGER.info(- "[whatsapp:%s] inbound message_id=%s chat_id=%s",- self.account_id,- message_id,- incoming.chat_id,- )- await self._dispatch(incoming)-- def _is_seen_message(self, message_id: str) -> bool:- return message_id in self._seen_message_ids-- def _prune_seen_message_ids(self) -> None:- cutoff = time.time() - WHATSAPP_MESSAGE_TTL_SECS- stale = [key for key, ts in self._seen_message_ids.items() if ts < cutoff]- for key in stale:- self._seen_message_ids.pop(key, None)-- def get_status_snapshot(self) -> dict[str, Any]:- return {- "account_id": self.account_id,- "session_key": self._session_key(),- "status": self._status,- "connected_at": self._connected_at or None,- "qr_updated_at": self._qr_updated_at or None,- "qr_data_url": self._qr_data_url or None,- "last_error": self._last_error or None,- "session_id": self._session_id or None,- "last_snapshot_at": self._last_snapshot_at or None,- "gateway_base_url": self._gateway_base_url() or None,- "event_cursor": self._event_cursor,- }---def _build(account_id: str, cfg: dict) -> Channel:- gateway_base_url = str(cfg.get("gateway_base_url") or cfg.get("gateway_url") or cfg.get("control_url") or _default_gateway_base_url())- gateway_token = str(cfg.get("gateway_token") or cfg.get("token") or "")- session_key = str(cfg.get("session_key") or cfg.get("session_id") or account_id)- timeout_secs = int(cfg.get("timeout_secs") or WHATSAPP_DEFAULT_TIMEOUT_SECS)- return WhatsAppChannel(- WhatsAppAccount(- account_id=account_id,- gateway_base_url=gateway_base_url,- gateway_token=gateway_token,- session_key=session_key,- timeout_secs=timeout_secs,- )- )---register_channel("whatsapp", _build)diff --git a/api/channels/whatsapp/gateway.py b/api/channels/whatsapp/gateway.pydeleted file mode 100644index a58149715d..0000000000--- a/api/channels/whatsapp/gateway.py+++ /dev/null@@ -1,171 +0,0 @@-from __future__ import annotations--import asyncio-import logging-import os-import shlex-import shutil-from dataclasses import dataclass-from pathlib import Path-from typing import Optional--LOGGER = logging.getLogger(__name__)-_missing_command_warned = False-_deps_install_warned = False---def _env_flag(name: str, default: bool = False) -> bool:- raw = os.getenv(name)- if raw is None:- return default- return raw.strip().lower() in {"1", "true", "yes", "on"}---def _default_gateway_command() -> list[str]:- raw = os.getenv("WHATSAPP_GATEWAY_COMMAND", "").strip()- if raw:- return shlex.split(raw)- gateway_entry = Path(__file__).resolve().parent / "gateway-node" / "index.js"- node = shutil.which("node")- if node and gateway_entry.exists():- return [node, str(gateway_entry)]- return []---def _gateway_dir() -> Path:- return Path(__file__).resolve().parent / "gateway-node"---@dataclass-class WhatsAppGatewayConfig:- command: list[str]- cwd: str- enabled: bool---class WhatsAppGatewayRuntime:- def __init__(self) -> None:- self._process: Optional[asyncio.subprocess.Process] = None- self._lock = asyncio.Lock()- self._install_lock = asyncio.Lock()- self._sync_generation = 0-- def _config(self) -> WhatsAppGatewayConfig:- workdir = os.getenv("WHATSAPP_GATEWAY_WORKDIR", "").strip()- return WhatsAppGatewayConfig(- command=_default_gateway_command(),- cwd=workdir or str(_gateway_dir()),- enabled=_env_flag("WHATSAPP_GATEWAY_ENABLED", True),- )-- def is_running(self) -> bool:- return bool(self._process and self._process.returncode is None)-- async def sync(self, enabled: bool) -> None:- cfg = self._config()- should_run = bool(enabled and cfg.enabled and cfg.command)- async with self._lock:- self._sync_generation += 1- generation = self._sync_generation- if not should_run:- await self._stop_locked()- return- if self.is_running():- return-- await self._ensure_dependencies(cfg)-- async with self._lock:- if generation != self._sync_generation:- return- if not should_run:- await self._stop_locked()- return- await self._start_locked(cfg)-- async def _ensure_dependencies(self, cfg: WhatsAppGatewayConfig) -> None:- global _deps_install_warned- if not _env_flag("WHATSAPP_GATEWAY_AUTO_INSTALL", True):- return-- async with self._install_lock:- gateway_dir = Path(cfg.cwd)- node_modules = gateway_dir / "node_modules"- if node_modules.exists():- return-- npm = shutil.which("npm")- if not npm:- if not _deps_install_warned:- LOGGER.warning("npm is not available; WhatsApp gateway dependencies cannot be installed automatically")- _deps_install_warned = True- return-- package_json = gateway_dir / "package.json"- if not package_json.exists():- LOGGER.warning("WhatsApp gateway package.json not found in %s", gateway_dir)- return-- LOGGER.info("installing WhatsApp gateway dependencies in %s", gateway_dir)- proc = await asyncio.create_subprocess_exec(- npm,- "install",- "--no-fund",- "--no-audit",- cwd=str(gateway_dir),- )- try:- code = await asyncio.wait_for(proc.wait(), timeout=300)- except asyncio.TimeoutError as ex:- proc.kill()- await proc.wait()- raise RuntimeError("npm install timed out after 300s") from ex- if code != 0:- raise RuntimeError(f"npm install failed with exit code {code}")- _deps_install_warned = False-- async def _start_locked(self, cfg: WhatsAppGatewayConfig) -> None:- global _missing_command_warned- if self.is_running():- return- if not cfg.command:- if not _missing_command_warned:- LOGGER.warning("WhatsApp gateway command is not configured; gateway will not start")- _missing_command_warned = True- return- _missing_command_warned = False-- env = os.environ.copy()- env.setdefault("PYTHONUNBUFFERED", "1")- LOGGER.info("starting WhatsApp gateway: %s", " ".join(cfg.command))- self._process = await asyncio.create_subprocess_exec(- *cfg.command,- cwd=cfg.cwd,- env=env,- )-- async def _stop_locked(self) -> None:- proc = self._process- if proc is None:- return-- if proc.returncode is None:- LOGGER.info("stopping WhatsApp gateway")- proc.terminate()- try:- await asyncio.wait_for(proc.wait(), timeout=10)- except asyncio.TimeoutError:- LOGGER.warning("WhatsApp gateway did not stop in time; killing it")- proc.kill()- await proc.wait()- except Exception:- LOGGER.debug("WhatsApp gateway stop failed", exc_info=True)-- self._process = None---_gateway_runtime = WhatsAppGatewayRuntime()---async def sync_whatsapp_gateway(enabled: bool) -> None:- await _gateway_runtime.sync(enabled)diff --git a/api/channels/whatsapp/gateway-node/README.md b/internal/channels/whatsapp-gateway/README.mdsimilarity index 100%rename from api/channels/whatsapp/gateway-node/README.mdrename to internal/channels/whatsapp-gateway/README.mddiff --git a/api/channels/whatsapp/gateway-node/index.js b/internal/channels/whatsapp-gateway/index.jssimilarity index 100%rename from api/channels/whatsapp/gateway-node/index.jsrename to internal/channels/whatsapp-gateway/index.jsdiff --git a/api/channels/whatsapp/gateway-node/package-lock.json b/internal/channels/whatsapp-gateway/package-lock.jsonsimilarity index 100%rename from api/channels/whatsapp/gateway-node/package-lock.jsonrename to internal/channels/whatsapp-gateway/package-lock.jsondiff --git a/api/channels/whatsapp/gateway-node/package.json b/internal/channels/whatsapp-gateway/package.jsonsimilarity index 100%rename from api/channels/whatsapp/gateway-node/package.jsonrename to internal/channels/whatsapp-gateway/package.jsondiff --git a/internal/channels/whatsapp_gateway.go b/internal/channels/whatsapp_gateway.goindex c20fd92990..1466291173 100644--- a/internal/channels/whatsapp_gateway.go+++ b/internal/channels/whatsapp_gateway.go@@ -94,7 +94,7 @@ func gatewayWorkdir() string {if !ok {return "."}- return filepath.Clean(filepath.Join(filepath.Dir(file), "../../api/channels/whatsapp/gateway-node"))+ return filepath.Clean(filepath.Join(filepath.Dir(file), "whatsapp-gateway"))}// start launches the gateway process if it is not already running.