Skip to content

LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows

UN-3439 [FIX] Accept wildcard subdomain origins in SocketIO and Django CORS (#1938)

ali5bc9e9c451e8
diff --git a/backend/backend/settings/base.py b/backend/backend/settings/base.py
index 2d493a91c..d954ab20f 100644
--- a/backend/backend/settings/base.py
+++ b/backend/backend/settings/base.py
@@ -12,11 +12,12 @@ https://docs.djangoproject.com/en/4.2/ref/settings/
import logging
import os
from pathlib import Path
-from urllib.parse import quote, urlparse
+from urllib.parse import quote
import httpx
from dotenv import find_dotenv, load_dotenv
from utils.common_utils import CommonUtils
+from utils.cors_origin import normalize_web_app_origin
missing_settings = []
@@ -68,10 +69,18 @@ WORKFLOW_ACTION_EXPIRATION_TIME_IN_SECOND = os.environ.get(
)
# Maximum number of files allowed per workflow page execution
WORKFLOW_PAGE_MAX_FILES = int(os.environ.get("WORKFLOW_PAGE_MAX_FILES", 2))
-WEB_APP_ORIGIN_URL = os.environ.get("WEB_APP_ORIGIN_URL", "http://localhost:3000")
-parsed_url = urlparse(WEB_APP_ORIGIN_URL)
-WEB_APP_ORIGIN_URL_WITH_WILD_CARD = f"{parsed_url.scheme}://*.{parsed_url.netloc}"
+(
+ WEB_APP_ORIGIN_URL,
+ WEB_APP_ORIGIN_URL_WITH_WILD_CARD,
+ _CORS_SUBDOMAIN_REGEX,
+) = normalize_web_app_origin(
+ os.environ.get("WEB_APP_ORIGIN_URL", "http://localhost:3000")
+)
CORS_ALLOWED_ORIGINS = [WEB_APP_ORIGIN_URL]
+# Wildcard subdomain regex consumed by django-cors-headers and (via the
+# RegexOrigin wrapper in utils/log_events.py) by the SocketIO engine.io
+# handshake — a single source of truth so HTTP and WS CORS cannot diverge.
+CORS_ALLOWED_ORIGIN_REGEXES = [_CORS_SUBDOMAIN_REGEX]
DJANGO_APP_BACKEND_URL = os.environ.get("DJANGO_APP_BACKEND_URL", "http://localhost:8000")
INTERNAL_SERVICE_API_KEY = os.environ.get("INTERNAL_SERVICE_API_KEY")
diff --git a/backend/utils/cors_origin.py b/backend/utils/cors_origin.py
new file mode 100644
index 000000000..aa640c6bc
--- /dev/null
+++ b/backend/utils/cors_origin.py
@@ -0,0 +1,81 @@
+"""CORS origin helpers used by Django settings and SocketIO log events.
+
+Kept free of Django imports so the matching/normalization logic can be unit
+tested without bootstrapping the full project.
+"""
+
+from __future__ import annotations
+
+import re
+from urllib.parse import urlparse
+
+
+class RegexOrigin:
+ """Origin pattern that compares to strings via regex match.
+
+ python-socketio enforces CORS with ``origin in allowed_origins`` during the
+ engine.io handshake — overriding ``__eq__`` lets a single list entry cover
+ a wildcard subdomain so bad origins are rejected before ``connect`` runs.
+
+ Instances are intentionally unhashable: a hashable object must satisfy
+ ``a == b ⇒ hash(a) == hash(b)``, but ``__eq__`` here is asymmetric across
+ types (matches many strings, hashes only one pattern). Any code that put
+ one in a ``set``/``frozenset`` would silently break the CORS gate, so
+ ``__hash__`` is disabled to fail loud at construction time instead.
+
+ ``fullmatch`` (not ``match``) is used so ``$`` doesn't permit a trailing
+ newline — defense in depth even though WSGI strips them upstream.
+ """
+
+ def __init__(self, pattern: str) -> None:
+ self._regex = re.compile(pattern)
+
+ def __eq__(self, other: object) -> bool:
+ if isinstance(other, str):
+ return self._regex.fullmatch(other) is not None
+ return NotImplemented
+
+ __hash__ = None # see class docstring
+
+
+def normalize_web_app_origin(env_value: str) -> tuple[str, str, str]:
+ """Parse and canonicalize ``WEB_APP_ORIGIN_URL`` for CORS/CSRF allow-lists.
+
+ Returns ``(origin, wildcard_origin, subdomain_regex)``:
+
+ - ``origin``: ``scheme://host[:port]`` form. Hostname is lowercased and
+ explicit default ports (:80 for http, :443 for https) are dropped, so
+ it matches what browsers serialize per RFC 6454.
+ - ``wildcard_origin``: same with a literal ``*.`` subdomain prefix, for
+ Django's ``CSRF_TRUSTED_ORIGINS`` (which fnmatches ``*``).
+ - ``subdomain_regex``: anchored pattern matching any subdomain of the
+ configured netloc, for ``CORS_ALLOWED_ORIGIN_REGEXES`` and SocketIO
+ via ``RegexOrigin``.
+
+ Raises:
+ ValueError: if the env value is not an http(s) URL with a host.
+ """
+ parsed = urlparse(env_value)
+ # `parsed.port` is a property that raises ValueError on malformed/out-of-range
+ # ports (e.g. `:abc`, `:99999`). Catch it here so misconfig surfaces with the
+ # same actionable message as every other validation failure.
+ try:
+ port = parsed.port
+ except ValueError as exc:
+ raise ValueError(
+ f"WEB_APP_ORIGIN_URL must be of the form http(s)://host[:port], "
+ f"got: {parsed.geturl()!r}"
+ ) from exc
+ if parsed.scheme not in {"http", "https"} or not parsed.hostname:
+ raise ValueError(
+ f"WEB_APP_ORIGIN_URL must be of the form http(s)://host[:port], "
+ f"got: {parsed.geturl()!r}"
+ )
+ default_port = {"http": 80, "https": 443}[parsed.scheme]
+ netloc = parsed.hostname
+ if port and port != default_port:
+ netloc = f"{netloc}:{port}"
+ origin = f"{parsed.scheme}://{netloc}"
+ wildcard = f"{parsed.scheme}://*.{netloc}"
+ subdomain_regex = rf"^{re.escape(parsed.scheme)}://[^/]+\.{re.escape(netloc)}$"
+ return origin, wildcard, subdomain_regex
diff --git a/backend/utils/log_events.py b/backend/utils/log_events.py
index f2ff72582..dcb66ed81 100644
--- a/backend/utils/log_events.py
+++ b/backend/utils/log_events.py
@@ -11,9 +11,15 @@ from unstract.core.cache.redis_client import create_redis_client
from unstract.core.data_models import LogDataDTO
from unstract.core.log_utils import get_validated_log_data, store_execution_log
from utils.constants import ExecutionLogConstants
+from utils.cors_origin import RegexOrigin
logger = logging.getLogger(__name__)
+
+_cors_allowed_origins: list[Any] = list(settings.CORS_ALLOWED_ORIGINS)
+for _pattern in getattr(settings, "CORS_ALLOWED_ORIGIN_REGEXES", []):
+ _cors_allowed_origins.append(RegexOrigin(_pattern))
+
_kombu_kwargs: dict[str, Any] = {"url": settings.SOCKET_IO_MANAGER_URL}
if getattr(settings, "SOCKET_IO_TRANSPORT_OPTIONS", None):
_kombu_kwargs["connection_options"] = {
@@ -23,7 +29,7 @@ if getattr(settings, "SOCKET_IO_TRANSPORT_OPTIONS", None):
sio = socketio.Server(
# Allowed values: {threading, eventlet, gevent, gevent_uwsgi}
async_mode="threading",
- cors_allowed_origins=settings.CORS_ALLOWED_ORIGINS,
+ cors_allowed_origins=_cors_allowed_origins,
logger=False,
engineio_logger=False,
always_connect=True,
diff --git a/backend/utils/tests/test_cors_origin.py b/backend/utils/tests/test_cors_origin.py
new file mode 100644
index 000000000..65628b440
--- /dev/null
+++ b/backend/utils/tests/test_cors_origin.py
@@ -0,0 +1,197 @@
+"""Regression tests for utils.cors_origin — the CORS origin matcher and
+URL normalizer that gate browser ``Origin`` headers on every Django request
+and SocketIO handshake.
+
+UN-3439: production socket connections silently failed for wildcard subdomains
+because python-socketio does exact-string comparison. These tests pin the
+contract so the next refactor can't reopen the hole.
+"""
+
+from __future__ import annotations
+
+import time
+
+import pytest
+
+from utils.cors_origin import RegexOrigin, normalize_web_app_origin
+
+
+class TestRegexOrigin:
+ def test_subdomain_matches(self):
+ ro = RegexOrigin(r"^https://[^/]+\.example\.com$")
+ assert ("https://app.example.com" == ro) is True
+ assert ("https://api.example.com" == ro) is True
+
+ def test_deep_subdomain_matches(self):
+ """Multi-level subdomains are accepted — DNS-owned by the same party."""
+ ro = RegexOrigin(r"^https://[^/]+\.example\.com$")
+ assert ("https://dev.env.example.com" == ro) is True
+
+ def test_apex_does_not_match(self):
+ """Apex is covered by the exact-match CORS_ALLOWED_ORIGINS entry, not
+ the wildcard regex."""
+ ro = RegexOrigin(r"^https://[^/]+\.example\.com$")
+ assert ("https://example.com" == ro) is False
+
+ def test_lookalike_rejected(self):
+ ro = RegexOrigin(r"^https://[^/]+\.example\.com$")
+ assert ("https://attacker-example.com" == ro) is False
+ assert ("https://example.com.attacker.com" == ro) is False
+ assert ("https://x.example.com.attacker.com" == ro) is False
+
+ def test_wrong_scheme_rejected(self):
+ ro = RegexOrigin(r"^https://[^/]+\.example\.com$")
+ # NOSONAR — the `http://` URL is intentional test data: we are
+ # asserting it is *rejected* by an https-scoped pattern.
+ assert ("http://app.example.com" == ro) is False # NOSONAR
+
+ def test_trailing_newline_rejected(self):
+ """``fullmatch`` (not ``match``) is required so ``$`` doesn't permit
+ a trailing ``\\n`` per Python regex semantics."""
+ ro = RegexOrigin(r"^https://[^/]+\.example\.com$")
+ assert ("https://app.example.com\n" == ro) is False
+
+ def test_in_operator_routes_to_eq(self):
+ """``origin in allowed_origins`` is exactly how python-socketio gates
+ the engine.io handshake — verify Python's reflected ``__eq__`` kicks in."""
+ allowed = [RegexOrigin(r"^https://[^/]+\.example\.com$")]
+ assert "https://app.example.com" in allowed
+ assert "https://evil.com" not in allowed
+
+ def test_non_string_returns_not_implemented(self):
+ """``__eq__`` must return the ``NotImplemented`` sentinel (not
+ ``False``) for non-strings so Python's reflected-equality protocol
+ can fall back to identity. Tested via direct dunder calls because
+ ``ro == None`` short-circuits before reaching ``__eq__``."""
+ ro = RegexOrigin(r"^x$")
+ assert ro.__eq__(None) is NotImplemented
+ assert ro.__eq__(42) is NotImplemented
+ assert ro.__eq__([]) is NotImplemented
+
+ def test_unhashable(self):
+ """``__hash__ = None`` prevents the equality/hash contract from being
+ violated if anyone wraps the allow-list in a ``set``/``frozenset``."""
+ ro = RegexOrigin(r"^x$")
+ with pytest.raises(TypeError):
+ hash(ro)
+ with pytest.raises(TypeError):
+ # `len({ro})` builds the set (calling __hash__) and consumes it via
+ # an explicit function call — keeps ruff from collapsing the
+ # statement and Sonar from flagging it as side-effect-free.
+ len({ro})
+
+ def test_no_redos(self):
+ """Pattern must complete on hostile input — ``[^/]+`` has no nested
+ quantifiers so backtracking is bounded. Threshold is generous (500ms)
+ so noisy CI runners don't flake; ReDoS would blow up by orders of
+ magnitude past this."""
+ ro = RegexOrigin(r"^https://[^/]+\.example\.com$")
+ hostile = "https://" + "a" * 10000 + ".evil.com"
+ start = time.perf_counter()
+ _ = hostile in [ro]
+ assert time.perf_counter() - start < 0.5
+
+
+class TestNormalizeWebAppOrigin:
+ def test_basic_https(self):
+ origin, wildcard, _ = normalize_web_app_origin("https://example.com")
+ assert origin == "https://example.com"
+ assert wildcard == "https://*.example.com"
+
+ def test_strips_trailing_slash(self):
+ origin, _, _ = normalize_web_app_origin("https://example.com/")
+ assert origin == "https://example.com"
+
+ def test_strips_path_and_query(self):
+ origin, _, _ = normalize_web_app_origin("https://example.com/path?q=1")
+ assert origin == "https://example.com"
+
+ def test_lowercases_hostname(self):
+ """Browsers serialize ``Origin`` with a lowercase host (RFC 6454);
+ django-cors-headers does case-sensitive string compare, so the env
+ value must be lowercased to match."""
+ origin, _, _ = normalize_web_app_origin("https://APP.EXAMPLE.COM")
+ assert origin == "https://app.example.com"
+
+ def test_drops_default_https_port(self):
+ """Browsers omit the explicit default port from ``Origin`` per
+ RFC 6454 — keeping ``:443`` would silently break exact match."""
+ origin, _, _ = normalize_web_app_origin("https://example.com:443")
+ assert origin == "https://example.com"
+
+ def test_drops_default_http_port(self):
+ # NOSONAR — `http://` URLs are intentional test data for the port
+ # normalization logic, not a runtime use of the insecure protocol.
+ origin, _, _ = normalize_web_app_origin("http://example.com:80") # NOSONAR
+ assert origin == "http://example.com" # NOSONAR
+
+ def test_keeps_non_default_port(self):
+ origin, wildcard, _ = normalize_web_app_origin("https://example.com:8443")
+ assert origin == "https://example.com:8443"
+ assert wildcard == "https://*.example.com:8443"
+
+ def test_localhost_default(self):
+ origin, _, _ = normalize_web_app_origin("http://localhost:3000")
+ assert origin == "http://localhost:3000"
+
+ @pytest.mark.parametrize(
+ "bad",
+ [
+ "",
+ "not-a-url",
+ "example.com", # missing scheme
+ "//example.com", # protocol-relative
+ "https://", # missing host
+ "ftp://example.com", # non-browser scheme # NOSONAR — test input asserting ftp is rejected
+ "ws://example.com", # not a top-level browser scheme
+ "https://example.com:abc", # malformed port — urlparse raises on .port access
+ "https://example.com:99999", # out-of-range port
+ ],
+ )
+ def test_rejects_misconfigured(self, bad):
+ """Fail fast at startup so misconfigured envs can't silently produce
+ CORS rules that match nothing real."""
+ with pytest.raises(ValueError, match="WEB_APP_ORIGIN_URL"):
+ normalize_web_app_origin(bad)
+
+
+class TestSubdomainRegexEndToEnd:
+ """End-to-end: the regex returned by ``normalize_web_app_origin`` is what
+ actually gates production. Verify via ``RegexOrigin`` (same path as
+ SocketIO uses)."""
+
+ def test_apex_env_accepts_subdomains(self):
+ _, _, pattern = normalize_web_app_origin("https://us-central.unstract.com")
+ ro = RegexOrigin(pattern)
+ # The exact failing origins from UN-3439:
+ assert "https://dev.env.us-central.unstract.com" in [ro]
+ assert "https://test.env.us-central.unstract.com" in [ro]
+
+ def test_apex_rejected_by_regex(self):
+ """Apex itself is *not* matched by the wildcard regex — that's the
+ exact-match CORS_ALLOWED_ORIGINS entry's job."""
+ _, _, pattern = normalize_web_app_origin("https://example.com")
+ ro = RegexOrigin(pattern)
+ assert "https://example.com" not in [ro]
+
+ @pytest.mark.parametrize(
+ "spoof",
+ [
+ "https://attacker-example.com",
+ "https://x.example.com.attacker.com",
+ "https://example.com.attacker.com",
+ "http://app.example.com", # wrong scheme # NOSONAR — test input asserting http is rejected
+ "https://app.example.com\n", # trailing newline
+ ],
+ )
+ def test_spoofed_origins_rejected(self, spoof):
+ _, _, pattern = normalize_web_app_origin("https://example.com")
+ ro = RegexOrigin(pattern)
+ assert spoof not in [ro], f"should reject {spoof!r}"
+
+ def test_uppercase_env_still_accepts_lowercase_origin(self):
+ """Browser sends lowercase even if the env was set with uppercase;
+ normalization must canonicalize before regex compilation."""
+ _, _, pattern = normalize_web_app_origin("https://APP.EXAMPLE.COM")
+ ro = RegexOrigin(pattern)
+ assert "https://sub.app.example.com" in [ro]