OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.
Python
fix: google to support WEBHOOK_BASE_URL (#1849)
Edwin Jose5ed2d6851638
diff --git a/src/config/settings.py b/src/config/settings.pyindex 846072821..33da28b16 100644--- a/src/config/settings.py+++ b/src/config/settings.py@@ -523,6 +523,9 @@ def is_no_auth_mode():# Webhook configuration - must be set to enable webhooksWEBHOOK_BASE_URL = os.getenv("WEBHOOK_BASE_URL") # No default - must be explicitly configured+# Legacy per-connector webhook URL override (takes precedence over WEBHOOK_BASE_URL)+GOOGLE_DRIVE_WEBHOOK_URL = os.getenv("GOOGLE_DRIVE_WEBHOOK_URL")+# OAuth callback broker URL -- when set, Google (and other providers) redirect# here instead of directly to the frontend. The broker then forwards to the# actual frontend origin that is carried in the OAuth state parameter.diff --git a/src/connectors/google_drive/connector.py b/src/connectors/google_drive/connector.pyindex 75126ff89..ebd0320e6 100644--- a/src/connectors/google_drive/connector.py+++ b/src/connectors/google_drive/connector.py@@ -409,6 +409,12 @@ class GoogleDriveConnector(BaseConnector):)continue+ if meta.get("trashed"):+ logger.debug(+ "[GoogleDrive] _iter_selected_items: file_id=%s is trashed, skipping", fid+ )+ continue+if meta.get("mimeType") == "application/vnd.google-apps.folder":logger.debug("[GoogleDrive] _iter_selected_items: %s (%s) is a folder, will expand",@@ -857,30 +863,57 @@ class GoogleDriveConnector(BaseConnector):)return doc+ def _resolve_webhook_address(self) -> str | None:+ """Resolve the Drive push notification URL from connection config or env."""+ webhook_url = self.config.get("webhook_url")+ if isinstance(webhook_url, str) and webhook_url.strip():+ return webhook_url.strip()++ from config.settings import GOOGLE_DRIVE_WEBHOOK_URL, WEBHOOK_BASE_URL++ legacy = GOOGLE_DRIVE_WEBHOOK_URL+ if legacy and legacy.strip():+ return legacy.strip()++ if WEBHOOK_BASE_URL:+ return f"{WEBHOOK_BASE_URL.rstrip('/')}/connectors/{self.CONNECTOR_TYPE}/webhook"++ webhook_address = getattr(self.cfg, "webhook_address", None)+ if isinstance(webhook_address, str) and webhook_address.strip():+ return webhook_address.strip()++ return None++ def extract_webhook_channel_id(+ self, payload: dict[str, Any], headers: dict[str, str]+ ) -> str | None:+ """Extract Google Drive channel ID from push notification headers."""+ normalized = {k.lower(): v for k, v in headers.items()}+ channel_id = normalized.get("x-goog-channel-id")+ if isinstance(channel_id, str) and channel_id.strip():+ return channel_id.strip()+ return None+async def setup_subscription(self) -> str:"""Start a Google Drive Changes API watch (webhook).Returns the channel ID (subscription ID) as a string.- Requires a webhook URL to be configured. This implementation looks for:- 1) self.cfg.webhook_address (preferred if you have it in your config dataclass)- 2) os.environ["GOOGLE_DRIVE_WEBHOOK_URL"]+ Webhook URL resolution order:+ 1) connection config ``webhook_url`` (from ``WEBHOOK_BASE_URL`` at connect time)+ 2) ``GOOGLE_DRIVE_WEBHOOK_URL`` env var (legacy override)+ 3) ``WEBHOOK_BASE_URL`` + ``/connectors/google_drive/webhook``"""- import os-# 1) Ensure we are authenticated and have a live Drive serviceok = await self.authenticate()if not ok:raise RuntimeError("GoogleDriveConnector.setup_subscription: not authenticated")- # 2) Resolve webhook address (no param in ABC, so pull from config/env)- webhook_address = getattr(self.cfg, "webhook_address", None) or os.getenv(- "GOOGLE_DRIVE_WEBHOOK_URL"- )+ webhook_address = self._resolve_webhook_address()if not webhook_address:raise RuntimeError("GoogleDriveConnector.setup_subscription: webhook URL not configured. "- "Set cfg.webhook_address or GOOGLE_DRIVE_WEBHOOK_URL."+ "Set WEBHOOK_BASE_URL or GOOGLE_DRIVE_WEBHOOK_URL.")# 3) Ensure we have a starting page token (checkpoint)diff --git a/src/connectors/onedrive/connector.py b/src/connectors/onedrive/connector.pyindex e563fa406..e14eb9d2a 100644--- a/src/connectors/onedrive/connector.py+++ b/src/connectors/onedrive/connector.py@@ -353,7 +353,9 @@ class OneDriveConnector(BaseConnector):subscription_data = {"changeType": "created,updated,deleted",- "notificationUrl": f"{webhook_url}/webhook/onedrive",+ # webhook_url is already the full endpoint+ # ({WEBHOOK_BASE_URL}/connectors/onedrive/webhook, set at connect time)+ "notificationUrl": webhook_url,"resource": resource,"expirationDateTime": self._get_subscription_expiry(),"clientState": "onedrive_personal",diff --git a/src/connectors/sharepoint/connector.py b/src/connectors/sharepoint/connector.pyindex ec4836376..09a2c3965 100644--- a/src/connectors/sharepoint/connector.py+++ b/src/connectors/sharepoint/connector.py@@ -391,7 +391,9 @@ class SharePointConnector(BaseConnector):subscription_data = {"changeType": "created,updated,deleted",- "notificationUrl": f"{webhook_url}/webhook/sharepoint",+ # webhook_url is already the full endpoint+ # ({WEBHOOK_BASE_URL}/connectors/sharepoint/webhook, set at connect time)+ "notificationUrl": webhook_url,"resource": resource,"expirationDateTime": self._get_subscription_expiry(),"clientState": f"sharepoint_{self.tenant_id}",diff --git a/tests/unit/connectors/test_connector_file_type_validation.py b/tests/unit/connectors/test_connector_file_type_validation.pyindex 12e48a154..7a90faf48 100644--- a/tests/unit/connectors/test_connector_file_type_validation.py+++ b/tests/unit/connectors/test_connector_file_type_validation.py@@ -141,6 +141,7 @@ async def test_connector_check_duplicates():response = await connector_check_duplicates(connector_type="onedrive",body=body,+ request=MagicMock(),connector_service=connector_service,session_manager=session_manager,user=user,diff --git a/tests/unit/connectors/test_langflow_connector_txt_to_md.py b/tests/unit/connectors/test_langflow_connector_txt_to_md.pyindex f57a7acd9..4ecc23d53 100644--- a/tests/unit/connectors/test_langflow_connector_txt_to_md.py+++ b/tests/unit/connectors/test_langflow_connector_txt_to_md.py@@ -39,6 +39,8 @@ def _make_service():langflow_service.merge_ui_ingest_settings_into_tweaks = MagicMock(return_value={})service.langflow_service = langflow_serviceservice.task_service = None+ # _get_effective_sync_jwt passes jwt_token through when no session manager+ service.session_manager = Nonereturn service, langflow_servicediff --git a/tests/unit/connectors/test_webhook_notification_url.py b/tests/unit/connectors/test_webhook_notification_url.pynew file mode 100644index 000000000..b9f51d815--- /dev/null+++ b/tests/unit/connectors/test_webhook_notification_url.py@@ -0,0 +1,205 @@+"""Webhook subscription URL correctness for all connectors.++Pins the fix for the doubled Microsoft Graph notificationUrl: the connection+config's ``webhook_url`` is already the full endpoint+(``{WEBHOOK_BASE_URL}/connectors/<type>/webhook``, set at connect time in+auth_service), so SharePoint/OneDrive must pass it to Graph verbatim. They+previously appended ``/webhook/<type>``, producing a URL with no registered+route — Graph's synchronous validation probe got a 404 and subscription+creation always failed.++Also pins Google Drive's webhook address resolution order+(config ``webhook_url`` → ``GOOGLE_DRIVE_WEBHOOK_URL`` setting →+``WEBHOOK_BASE_URL`` + path) and the AWS S3 no-op stubs.+"""++import sys+from pathlib import Path+from unittest.mock import AsyncMock++import pytest++ROOT = Path(__file__).resolve().parent.parent.parent.parent+SRC = ROOT / "src"+if str(SRC) not in sys.path:+ sys.path.insert(0, str(SRC))+++class _FakeResponse:+ def __init__(self, payload: dict):+ self._payload = payload++ def raise_for_status(self):+ pass++ def json(self) -> dict:+ return self._payload+++class _FakeAsyncClient:+ """Stands in for httpx.AsyncClient; captures the Graph subscription POST."""++ def __init__(self, captured: dict):+ self._captured = captured++ def __call__(self, *args, **kwargs):+ # Connector code instantiates httpx.AsyncClient(); return ourselves.+ return self++ async def __aenter__(self):+ return self++ async def __aexit__(self, *exc):+ return False++ async def post(self, url, json=None, headers=None, timeout=None):+ self._captured["url"] = url+ self._captured["json"] = json+ return _FakeResponse({"id": "sub-123"})+++class _FakeOAuth:+ def get_access_token(self) -> str:+ return "access-token"+++GRAPH_CONNECTORS = [+ ("connectors.sharepoint.connector", "SharePointConnector", "sharepoint"),+ ("connectors.onedrive.connector", "OneDriveConnector", "onedrive"),+]+++def _graph_connector(module_path: str, cls_name: str, tmp_path, webhook_url: str | None):+ import importlib++ cls = getattr(importlib.import_module(module_path), cls_name)+ config = {"token_file": str(tmp_path / "token.json")}+ if webhook_url:+ config["webhook_url"] = webhook_url+ connector = cls(config)+ connector.authenticate = AsyncMock(return_value=True)+ connector.oauth = _FakeOAuth()+ return connector+++# ---------------------------------------------------------------------------+# SharePoint / OneDrive — Graph notificationUrl+# ---------------------------------------------------------------------------++[email protected]("module_path,cls_name,connector_type", GRAPH_CONNECTORS)+async def test_graph_notification_url_is_config_webhook_url_verbatim(+ tmp_path, monkeypatch, module_path, cls_name, connector_type+):+ import httpx++ webhook_url = f"https://api.example.com/connectors/{connector_type}/webhook"+ connector = _graph_connector(module_path, cls_name, tmp_path, webhook_url)++ captured = {}+ monkeypatch.setattr(httpx, "AsyncClient", _FakeAsyncClient(captured))++ subscription_id = await connector.setup_subscription()++ assert subscription_id == "sub-123"+ assert captured["url"].endswith("/subscriptions")+ body = captured["json"]+ assert body["notificationUrl"] == webhook_url+ # Regression: must not re-append a /webhook/<type> segment to the+ # already-complete endpoint (yields a 404 route, Graph rejects it).+ assert f"/webhook/{connector_type}" not in body["notificationUrl"]++[email protected]("module_path,cls_name,connector_type", GRAPH_CONNECTORS)+async def test_graph_subscription_skipped_without_webhook_url(+ tmp_path, module_path, cls_name, connector_type+):+ connector = _graph_connector(module_path, cls_name, tmp_path, webhook_url=None)++ assert await connector.setup_subscription() == "no-webhook-configured"+ connector.authenticate.assert_not_awaited()+++# ---------------------------------------------------------------------------+# Google Drive — _resolve_webhook_address resolution order+# ---------------------------------------------------------------------------+++def _gdrive_connector(config: dict):+ """Build a minimal GoogleDriveConnector via __new__ (no real credentials)."""+ from connectors.google_drive.connector import GoogleDriveConfig, GoogleDriveConnector++ connector = GoogleDriveConnector.__new__(GoogleDriveConnector)+ connector.config = config+ connector.cfg = GoogleDriveConfig(+ client_id="fake-client-id",+ client_secret="fake-client-secret",+ token_file="/tmp/fake-token.json",+ )+ return connector+++def _set_webhook_settings(monkeypatch, legacy: str | None, base: str | None):+ import config.settings as settings++ monkeypatch.setattr(settings, "GOOGLE_DRIVE_WEBHOOK_URL", legacy)+ monkeypatch.setattr(settings, "WEBHOOK_BASE_URL", base)+++def test_google_drive_config_webhook_url_wins(monkeypatch):+ _set_webhook_settings(+ monkeypatch, "https://legacy.example.com/hook", "https://base.example.com"+ )+ connector = _gdrive_connector(+ {"webhook_url": " https://api.example.com/connectors/google_drive/webhook "}+ )++ assert (+ connector._resolve_webhook_address()+ == "https://api.example.com/connectors/google_drive/webhook"+ )+++def test_google_drive_legacy_setting_overrides_base_url(monkeypatch):+ _set_webhook_settings(+ monkeypatch, "https://legacy.example.com/hook", "https://base.example.com"+ )+ connector = _gdrive_connector({})++ assert connector._resolve_webhook_address() == "https://legacy.example.com/hook"+++def test_google_drive_falls_back_to_webhook_base_url(monkeypatch):+ _set_webhook_settings(monkeypatch, None, "https://base.example.com/")+ connector = _gdrive_connector({})++ assert (+ connector._resolve_webhook_address()+ == "https://base.example.com/connectors/google_drive/webhook"+ )+++def test_google_drive_returns_none_when_nothing_configured(monkeypatch):+ _set_webhook_settings(monkeypatch, None, None)+ connector = _gdrive_connector({})++ assert connector._resolve_webhook_address() is None+++# ---------------------------------------------------------------------------+# AWS S3 — subscriptions are explicit no-ops+# ---------------------------------------------------------------------------+++async def test_aws_s3_subscription_is_noop():+ from connectors.aws_s3.connector import S3Connector++ connector = S3Connector({"bucket_names": ["bucket"]})++ assert await connector.setup_subscription() == ""+ assert await connector.cleanup_subscription("") is True+ assert connector.extract_webhook_channel_id({}, {}) is None+ assert await connector.handle_webhook({"value": []}) == []