Skip to content

OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.

Python

fix: google to support WEBHOOK_BASE_URL (#1849)

Edwin Jose5ed2d6851638
diff --git a/src/config/settings.py b/src/config/settings.py
index 846072821..33da28b16 100644
--- a/src/config/settings.py
+++ b/src/config/settings.py
@@ -523,6 +523,9 @@ def is_no_auth_mode():
# Webhook configuration - must be set to enable webhooks
WEBHOOK_BASE_URL = os.getenv("WEBHOOK_BASE_URL") # No default - must be explicitly configured
+# Legacy per-connector webhook URL override (takes precedence over WEBHOOK_BASE_URL)
+GOOGLE_DRIVE_WEBHOOK_URL = os.getenv("GOOGLE_DRIVE_WEBHOOK_URL")
+
# OAuth callback broker URL -- when set, Google (and other providers) redirect
# here instead of directly to the frontend. The broker then forwards to the
# actual frontend origin that is carried in the OAuth state parameter.
diff --git a/src/connectors/google_drive/connector.py b/src/connectors/google_drive/connector.py
index 75126ff89..ebd0320e6 100644
--- a/src/connectors/google_drive/connector.py
+++ b/src/connectors/google_drive/connector.py
@@ -409,6 +409,12 @@ class GoogleDriveConnector(BaseConnector):
)
continue
+ if meta.get("trashed"):
+ logger.debug(
+ "[GoogleDrive] _iter_selected_items: file_id=%s is trashed, skipping", fid
+ )
+ continue
+
if meta.get("mimeType") == "application/vnd.google-apps.folder":
logger.debug(
"[GoogleDrive] _iter_selected_items: %s (%s) is a folder, will expand",
@@ -857,30 +863,57 @@ class GoogleDriveConnector(BaseConnector):
)
return doc
+ def _resolve_webhook_address(self) -> str | None:
+ """Resolve the Drive push notification URL from connection config or env."""
+ webhook_url = self.config.get("webhook_url")
+ if isinstance(webhook_url, str) and webhook_url.strip():
+ return webhook_url.strip()
+
+ from config.settings import GOOGLE_DRIVE_WEBHOOK_URL, WEBHOOK_BASE_URL
+
+ legacy = GOOGLE_DRIVE_WEBHOOK_URL
+ if legacy and legacy.strip():
+ return legacy.strip()
+
+ if WEBHOOK_BASE_URL:
+ return f"{WEBHOOK_BASE_URL.rstrip('/')}/connectors/{self.CONNECTOR_TYPE}/webhook"
+
+ webhook_address = getattr(self.cfg, "webhook_address", None)
+ if isinstance(webhook_address, str) and webhook_address.strip():
+ return webhook_address.strip()
+
+ return None
+
+ def extract_webhook_channel_id(
+ self, payload: dict[str, Any], headers: dict[str, str]
+ ) -> str | None:
+ """Extract Google Drive channel ID from push notification headers."""
+ normalized = {k.lower(): v for k, v in headers.items()}
+ channel_id = normalized.get("x-goog-channel-id")
+ if isinstance(channel_id, str) and channel_id.strip():
+ return channel_id.strip()
+ return None
+
async def setup_subscription(self) -> str:
"""
Start a Google Drive Changes API watch (webhook).
Returns the channel ID (subscription ID) as a string.
- Requires a webhook URL to be configured. This implementation looks for:
- 1) self.cfg.webhook_address (preferred if you have it in your config dataclass)
- 2) os.environ["GOOGLE_DRIVE_WEBHOOK_URL"]
+ Webhook URL resolution order:
+ 1) connection config ``webhook_url`` (from ``WEBHOOK_BASE_URL`` at connect time)
+ 2) ``GOOGLE_DRIVE_WEBHOOK_URL`` env var (legacy override)
+ 3) ``WEBHOOK_BASE_URL`` + ``/connectors/google_drive/webhook``
"""
- import os
-
# 1) Ensure we are authenticated and have a live Drive service
ok = await self.authenticate()
if not ok:
raise RuntimeError("GoogleDriveConnector.setup_subscription: not authenticated")
- # 2) Resolve webhook address (no param in ABC, so pull from config/env)
- webhook_address = getattr(self.cfg, "webhook_address", None) or os.getenv(
- "GOOGLE_DRIVE_WEBHOOK_URL"
- )
+ webhook_address = self._resolve_webhook_address()
if not webhook_address:
raise RuntimeError(
"GoogleDriveConnector.setup_subscription: webhook URL not configured. "
- "Set cfg.webhook_address or GOOGLE_DRIVE_WEBHOOK_URL."
+ "Set WEBHOOK_BASE_URL or GOOGLE_DRIVE_WEBHOOK_URL."
)
# 3) Ensure we have a starting page token (checkpoint)
diff --git a/src/connectors/onedrive/connector.py b/src/connectors/onedrive/connector.py
index e563fa406..e14eb9d2a 100644
--- a/src/connectors/onedrive/connector.py
+++ b/src/connectors/onedrive/connector.py
@@ -353,7 +353,9 @@ class OneDriveConnector(BaseConnector):
subscription_data = {
"changeType": "created,updated,deleted",
- "notificationUrl": f"{webhook_url}/webhook/onedrive",
+ # webhook_url is already the full endpoint
+ # ({WEBHOOK_BASE_URL}/connectors/onedrive/webhook, set at connect time)
+ "notificationUrl": webhook_url,
"resource": resource,
"expirationDateTime": self._get_subscription_expiry(),
"clientState": "onedrive_personal",
diff --git a/src/connectors/sharepoint/connector.py b/src/connectors/sharepoint/connector.py
index ec4836376..09a2c3965 100644
--- a/src/connectors/sharepoint/connector.py
+++ b/src/connectors/sharepoint/connector.py
@@ -391,7 +391,9 @@ class SharePointConnector(BaseConnector):
subscription_data = {
"changeType": "created,updated,deleted",
- "notificationUrl": f"{webhook_url}/webhook/sharepoint",
+ # webhook_url is already the full endpoint
+ # ({WEBHOOK_BASE_URL}/connectors/sharepoint/webhook, set at connect time)
+ "notificationUrl": webhook_url,
"resource": resource,
"expirationDateTime": self._get_subscription_expiry(),
"clientState": f"sharepoint_{self.tenant_id}",
diff --git a/tests/unit/connectors/test_connector_file_type_validation.py b/tests/unit/connectors/test_connector_file_type_validation.py
index 12e48a154..7a90faf48 100644
--- a/tests/unit/connectors/test_connector_file_type_validation.py
+++ b/tests/unit/connectors/test_connector_file_type_validation.py
@@ -141,6 +141,7 @@ async def test_connector_check_duplicates():
response = await connector_check_duplicates(
connector_type="onedrive",
body=body,
+ request=MagicMock(),
connector_service=connector_service,
session_manager=session_manager,
user=user,
diff --git a/tests/unit/connectors/test_langflow_connector_txt_to_md.py b/tests/unit/connectors/test_langflow_connector_txt_to_md.py
index f57a7acd9..4ecc23d53 100644
--- a/tests/unit/connectors/test_langflow_connector_txt_to_md.py
+++ b/tests/unit/connectors/test_langflow_connector_txt_to_md.py
@@ -39,6 +39,8 @@ def _make_service():
langflow_service.merge_ui_ingest_settings_into_tweaks = MagicMock(return_value={})
service.langflow_service = langflow_service
service.task_service = None
+ # _get_effective_sync_jwt passes jwt_token through when no session manager
+ service.session_manager = None
return service, langflow_service
diff --git a/tests/unit/connectors/test_webhook_notification_url.py b/tests/unit/connectors/test_webhook_notification_url.py
new file mode 100644
index 000000000..b9f51d815
--- /dev/null
+++ b/tests/unit/connectors/test_webhook_notification_url.py
@@ -0,0 +1,205 @@
+"""Webhook subscription URL correctness for all connectors.
+
+Pins the fix for the doubled Microsoft Graph notificationUrl: the connection
+config's ``webhook_url`` is already the full endpoint
+(``{WEBHOOK_BASE_URL}/connectors/<type>/webhook``, set at connect time in
+auth_service), so SharePoint/OneDrive must pass it to Graph verbatim. They
+previously appended ``/webhook/<type>``, producing a URL with no registered
+route — Graph's synchronous validation probe got a 404 and subscription
+creation always failed.
+
+Also pins Google Drive's webhook address resolution order
+(config ``webhook_url`` → ``GOOGLE_DRIVE_WEBHOOK_URL`` setting →
+``WEBHOOK_BASE_URL`` + path) and the AWS S3 no-op stubs.
+"""
+
+import sys
+from pathlib import Path
+from unittest.mock import AsyncMock
+
+import pytest
+
+ROOT = Path(__file__).resolve().parent.parent.parent.parent
+SRC = ROOT / "src"
+if str(SRC) not in sys.path:
+ sys.path.insert(0, str(SRC))
+
+
+class _FakeResponse:
+ def __init__(self, payload: dict):
+ self._payload = payload
+
+ def raise_for_status(self):
+ pass
+
+ def json(self) -> dict:
+ return self._payload
+
+
+class _FakeAsyncClient:
+ """Stands in for httpx.AsyncClient; captures the Graph subscription POST."""
+
+ def __init__(self, captured: dict):
+ self._captured = captured
+
+ def __call__(self, *args, **kwargs):
+ # Connector code instantiates httpx.AsyncClient(); return ourselves.
+ return self
+
+ async def __aenter__(self):
+ return self
+
+ async def __aexit__(self, *exc):
+ return False
+
+ async def post(self, url, json=None, headers=None, timeout=None):
+ self._captured["url"] = url
+ self._captured["json"] = json
+ return _FakeResponse({"id": "sub-123"})
+
+
+class _FakeOAuth:
+ def get_access_token(self) -> str:
+ return "access-token"
+
+
+GRAPH_CONNECTORS = [
+ ("connectors.sharepoint.connector", "SharePointConnector", "sharepoint"),
+ ("connectors.onedrive.connector", "OneDriveConnector", "onedrive"),
+]
+
+
+def _graph_connector(module_path: str, cls_name: str, tmp_path, webhook_url: str | None):
+ import importlib
+
+ cls = getattr(importlib.import_module(module_path), cls_name)
+ config = {"token_file": str(tmp_path / "token.json")}
+ if webhook_url:
+ config["webhook_url"] = webhook_url
+ connector = cls(config)
+ connector.authenticate = AsyncMock(return_value=True)
+ connector.oauth = _FakeOAuth()
+ return connector
+
+
+# ---------------------------------------------------------------------------
+# SharePoint / OneDrive — Graph notificationUrl
+# ---------------------------------------------------------------------------
+
+
[email protected]("module_path,cls_name,connector_type", GRAPH_CONNECTORS)
+async def test_graph_notification_url_is_config_webhook_url_verbatim(
+ tmp_path, monkeypatch, module_path, cls_name, connector_type
+):
+ import httpx
+
+ webhook_url = f"https://api.example.com/connectors/{connector_type}/webhook"
+ connector = _graph_connector(module_path, cls_name, tmp_path, webhook_url)
+
+ captured = {}
+ monkeypatch.setattr(httpx, "AsyncClient", _FakeAsyncClient(captured))
+
+ subscription_id = await connector.setup_subscription()
+
+ assert subscription_id == "sub-123"
+ assert captured["url"].endswith("/subscriptions")
+ body = captured["json"]
+ assert body["notificationUrl"] == webhook_url
+ # Regression: must not re-append a /webhook/<type> segment to the
+ # already-complete endpoint (yields a 404 route, Graph rejects it).
+ assert f"/webhook/{connector_type}" not in body["notificationUrl"]
+
+
[email protected]("module_path,cls_name,connector_type", GRAPH_CONNECTORS)
+async def test_graph_subscription_skipped_without_webhook_url(
+ tmp_path, module_path, cls_name, connector_type
+):
+ connector = _graph_connector(module_path, cls_name, tmp_path, webhook_url=None)
+
+ assert await connector.setup_subscription() == "no-webhook-configured"
+ connector.authenticate.assert_not_awaited()
+
+
+# ---------------------------------------------------------------------------
+# Google Drive — _resolve_webhook_address resolution order
+# ---------------------------------------------------------------------------
+
+
+def _gdrive_connector(config: dict):
+ """Build a minimal GoogleDriveConnector via __new__ (no real credentials)."""
+ from connectors.google_drive.connector import GoogleDriveConfig, GoogleDriveConnector
+
+ connector = GoogleDriveConnector.__new__(GoogleDriveConnector)
+ connector.config = config
+ connector.cfg = GoogleDriveConfig(
+ client_id="fake-client-id",
+ client_secret="fake-client-secret",
+ token_file="/tmp/fake-token.json",
+ )
+ return connector
+
+
+def _set_webhook_settings(monkeypatch, legacy: str | None, base: str | None):
+ import config.settings as settings
+
+ monkeypatch.setattr(settings, "GOOGLE_DRIVE_WEBHOOK_URL", legacy)
+ monkeypatch.setattr(settings, "WEBHOOK_BASE_URL", base)
+
+
+def test_google_drive_config_webhook_url_wins(monkeypatch):
+ _set_webhook_settings(
+ monkeypatch, "https://legacy.example.com/hook", "https://base.example.com"
+ )
+ connector = _gdrive_connector(
+ {"webhook_url": " https://api.example.com/connectors/google_drive/webhook "}
+ )
+
+ assert (
+ connector._resolve_webhook_address()
+ == "https://api.example.com/connectors/google_drive/webhook"
+ )
+
+
+def test_google_drive_legacy_setting_overrides_base_url(monkeypatch):
+ _set_webhook_settings(
+ monkeypatch, "https://legacy.example.com/hook", "https://base.example.com"
+ )
+ connector = _gdrive_connector({})
+
+ assert connector._resolve_webhook_address() == "https://legacy.example.com/hook"
+
+
+def test_google_drive_falls_back_to_webhook_base_url(monkeypatch):
+ _set_webhook_settings(monkeypatch, None, "https://base.example.com/")
+ connector = _gdrive_connector({})
+
+ assert (
+ connector._resolve_webhook_address()
+ == "https://base.example.com/connectors/google_drive/webhook"
+ )
+
+
+def test_google_drive_returns_none_when_nothing_configured(monkeypatch):
+ _set_webhook_settings(monkeypatch, None, None)
+ connector = _gdrive_connector({})
+
+ assert connector._resolve_webhook_address() is None
+
+
+# ---------------------------------------------------------------------------
+# AWS S3 — subscriptions are explicit no-ops
+# ---------------------------------------------------------------------------
+
+
+async def test_aws_s3_subscription_is_noop():
+ from connectors.aws_s3.connector import S3Connector
+
+ connector = S3Connector({"bucket_names": ["bucket"]})
+
+ assert await connector.setup_subscription() == ""
+ assert await connector.cleanup_subscription("") is True
+ assert connector.extract_webhook_channel_id({}, {}) is None
+ assert await connector.handle_webhook({"value": []}) == []