LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows
Python ai-agentsdata-engineeringdocument-aigenerative-aiidpjson-extractionllmmcp-serverocrpdf-extractionprompt-engineeringstructured-output
UN-3439 [FIX] Accept wildcard subdomain origins in SocketIO and Django CORS (#1938)
ali5bc9e9c451e8
diff --git a/backend/backend/settings/base.py b/backend/backend/settings/base.pyindex 2d493a91c..d954ab20f 100644--- a/backend/backend/settings/base.py+++ b/backend/backend/settings/base.py@@ -12,11 +12,12 @@ https://docs.djangoproject.com/en/4.2/ref/settings/import loggingimport osfrom pathlib import Path-from urllib.parse import quote, urlparse+from urllib.parse import quoteimport httpxfrom dotenv import find_dotenv, load_dotenvfrom utils.common_utils import CommonUtils+from utils.cors_origin import normalize_web_app_originmissing_settings = []@@ -68,10 +69,18 @@ WORKFLOW_ACTION_EXPIRATION_TIME_IN_SECOND = os.environ.get()# Maximum number of files allowed per workflow page executionWORKFLOW_PAGE_MAX_FILES = int(os.environ.get("WORKFLOW_PAGE_MAX_FILES", 2))-WEB_APP_ORIGIN_URL = os.environ.get("WEB_APP_ORIGIN_URL", "http://localhost:3000")-parsed_url = urlparse(WEB_APP_ORIGIN_URL)-WEB_APP_ORIGIN_URL_WITH_WILD_CARD = f"{parsed_url.scheme}://*.{parsed_url.netloc}"+(+ WEB_APP_ORIGIN_URL,+ WEB_APP_ORIGIN_URL_WITH_WILD_CARD,+ _CORS_SUBDOMAIN_REGEX,+) = normalize_web_app_origin(+ os.environ.get("WEB_APP_ORIGIN_URL", "http://localhost:3000")+)CORS_ALLOWED_ORIGINS = [WEB_APP_ORIGIN_URL]+# Wildcard subdomain regex consumed by django-cors-headers and (via the+# RegexOrigin wrapper in utils/log_events.py) by the SocketIO engine.io+# handshake — a single source of truth so HTTP and WS CORS cannot diverge.+CORS_ALLOWED_ORIGIN_REGEXES = [_CORS_SUBDOMAIN_REGEX]DJANGO_APP_BACKEND_URL = os.environ.get("DJANGO_APP_BACKEND_URL", "http://localhost:8000")INTERNAL_SERVICE_API_KEY = os.environ.get("INTERNAL_SERVICE_API_KEY")diff --git a/backend/utils/cors_origin.py b/backend/utils/cors_origin.pynew file mode 100644index 000000000..aa640c6bc--- /dev/null+++ b/backend/utils/cors_origin.py@@ -0,0 +1,81 @@+"""CORS origin helpers used by Django settings and SocketIO log events.++Kept free of Django imports so the matching/normalization logic can be unit+tested without bootstrapping the full project.+"""++from __future__ import annotations++import re+from urllib.parse import urlparse+++class RegexOrigin:+ """Origin pattern that compares to strings via regex match.++ python-socketio enforces CORS with ``origin in allowed_origins`` during the+ engine.io handshake — overriding ``__eq__`` lets a single list entry cover+ a wildcard subdomain so bad origins are rejected before ``connect`` runs.++ Instances are intentionally unhashable: a hashable object must satisfy+ ``a == b ⇒ hash(a) == hash(b)``, but ``__eq__`` here is asymmetric across+ types (matches many strings, hashes only one pattern). Any code that put+ one in a ``set``/``frozenset`` would silently break the CORS gate, so+ ``__hash__`` is disabled to fail loud at construction time instead.++ ``fullmatch`` (not ``match``) is used so ``$`` doesn't permit a trailing+ newline — defense in depth even though WSGI strips them upstream.+ """++ def __init__(self, pattern: str) -> None:+ self._regex = re.compile(pattern)++ def __eq__(self, other: object) -> bool:+ if isinstance(other, str):+ return self._regex.fullmatch(other) is not None+ return NotImplemented++ __hash__ = None # see class docstring+++def normalize_web_app_origin(env_value: str) -> tuple[str, str, str]:+ """Parse and canonicalize ``WEB_APP_ORIGIN_URL`` for CORS/CSRF allow-lists.++ Returns ``(origin, wildcard_origin, subdomain_regex)``:++ - ``origin``: ``scheme://host[:port]`` form. Hostname is lowercased and+ explicit default ports (:80 for http, :443 for https) are dropped, so+ it matches what browsers serialize per RFC 6454.+ - ``wildcard_origin``: same with a literal ``*.`` subdomain prefix, for+ Django's ``CSRF_TRUSTED_ORIGINS`` (which fnmatches ``*``).+ - ``subdomain_regex``: anchored pattern matching any subdomain of the+ configured netloc, for ``CORS_ALLOWED_ORIGIN_REGEXES`` and SocketIO+ via ``RegexOrigin``.++ Raises:+ ValueError: if the env value is not an http(s) URL with a host.+ """+ parsed = urlparse(env_value)+ # `parsed.port` is a property that raises ValueError on malformed/out-of-range+ # ports (e.g. `:abc`, `:99999`). Catch it here so misconfig surfaces with the+ # same actionable message as every other validation failure.+ try:+ port = parsed.port+ except ValueError as exc:+ raise ValueError(+ f"WEB_APP_ORIGIN_URL must be of the form http(s)://host[:port], "+ f"got: {parsed.geturl()!r}"+ ) from exc+ if parsed.scheme not in {"http", "https"} or not parsed.hostname:+ raise ValueError(+ f"WEB_APP_ORIGIN_URL must be of the form http(s)://host[:port], "+ f"got: {parsed.geturl()!r}"+ )+ default_port = {"http": 80, "https": 443}[parsed.scheme]+ netloc = parsed.hostname+ if port and port != default_port:+ netloc = f"{netloc}:{port}"+ origin = f"{parsed.scheme}://{netloc}"+ wildcard = f"{parsed.scheme}://*.{netloc}"+ subdomain_regex = rf"^{re.escape(parsed.scheme)}://[^/]+\.{re.escape(netloc)}$"+ return origin, wildcard, subdomain_regexdiff --git a/backend/utils/log_events.py b/backend/utils/log_events.pyindex f2ff72582..dcb66ed81 100644--- a/backend/utils/log_events.py+++ b/backend/utils/log_events.py@@ -11,9 +11,15 @@ from unstract.core.cache.redis_client import create_redis_clientfrom unstract.core.data_models import LogDataDTOfrom unstract.core.log_utils import get_validated_log_data, store_execution_logfrom utils.constants import ExecutionLogConstants+from utils.cors_origin import RegexOriginlogger = logging.getLogger(__name__)++_cors_allowed_origins: list[Any] = list(settings.CORS_ALLOWED_ORIGINS)+for _pattern in getattr(settings, "CORS_ALLOWED_ORIGIN_REGEXES", []):+ _cors_allowed_origins.append(RegexOrigin(_pattern))+_kombu_kwargs: dict[str, Any] = {"url": settings.SOCKET_IO_MANAGER_URL}if getattr(settings, "SOCKET_IO_TRANSPORT_OPTIONS", None):_kombu_kwargs["connection_options"] = {@@ -23,7 +29,7 @@ if getattr(settings, "SOCKET_IO_TRANSPORT_OPTIONS", None):sio = socketio.Server(# Allowed values: {threading, eventlet, gevent, gevent_uwsgi}async_mode="threading",- cors_allowed_origins=settings.CORS_ALLOWED_ORIGINS,+ cors_allowed_origins=_cors_allowed_origins,logger=False,engineio_logger=False,always_connect=True,diff --git a/backend/utils/tests/test_cors_origin.py b/backend/utils/tests/test_cors_origin.pynew file mode 100644index 000000000..65628b440--- /dev/null+++ b/backend/utils/tests/test_cors_origin.py@@ -0,0 +1,197 @@+"""Regression tests for utils.cors_origin — the CORS origin matcher and+URL normalizer that gate browser ``Origin`` headers on every Django request+and SocketIO handshake.++UN-3439: production socket connections silently failed for wildcard subdomains+because python-socketio does exact-string comparison. These tests pin the+contract so the next refactor can't reopen the hole.+"""++from __future__ import annotations++import time++import pytest++from utils.cors_origin import RegexOrigin, normalize_web_app_origin+++class TestRegexOrigin:+ def test_subdomain_matches(self):+ ro = RegexOrigin(r"^https://[^/]+\.example\.com$")+ assert ("https://app.example.com" == ro) is True+ assert ("https://api.example.com" == ro) is True++ def test_deep_subdomain_matches(self):+ """Multi-level subdomains are accepted — DNS-owned by the same party."""+ ro = RegexOrigin(r"^https://[^/]+\.example\.com$")+ assert ("https://dev.env.example.com" == ro) is True++ def test_apex_does_not_match(self):+ """Apex is covered by the exact-match CORS_ALLOWED_ORIGINS entry, not+ the wildcard regex."""+ ro = RegexOrigin(r"^https://[^/]+\.example\.com$")+ assert ("https://example.com" == ro) is False++ def test_lookalike_rejected(self):+ ro = RegexOrigin(r"^https://[^/]+\.example\.com$")+ assert ("https://attacker-example.com" == ro) is False+ assert ("https://example.com.attacker.com" == ro) is False+ assert ("https://x.example.com.attacker.com" == ro) is False++ def test_wrong_scheme_rejected(self):+ ro = RegexOrigin(r"^https://[^/]+\.example\.com$")+ # NOSONAR — the `http://` URL is intentional test data: we are+ # asserting it is *rejected* by an https-scoped pattern.+ assert ("http://app.example.com" == ro) is False # NOSONAR++ def test_trailing_newline_rejected(self):+ """``fullmatch`` (not ``match``) is required so ``$`` doesn't permit+ a trailing ``\\n`` per Python regex semantics."""+ ro = RegexOrigin(r"^https://[^/]+\.example\.com$")+ assert ("https://app.example.com\n" == ro) is False++ def test_in_operator_routes_to_eq(self):+ """``origin in allowed_origins`` is exactly how python-socketio gates+ the engine.io handshake — verify Python's reflected ``__eq__`` kicks in."""+ allowed = [RegexOrigin(r"^https://[^/]+\.example\.com$")]+ assert "https://app.example.com" in allowed+ assert "https://evil.com" not in allowed++ def test_non_string_returns_not_implemented(self):+ """``__eq__`` must return the ``NotImplemented`` sentinel (not+ ``False``) for non-strings so Python's reflected-equality protocol+ can fall back to identity. Tested via direct dunder calls because+ ``ro == None`` short-circuits before reaching ``__eq__``."""+ ro = RegexOrigin(r"^x$")+ assert ro.__eq__(None) is NotImplemented+ assert ro.__eq__(42) is NotImplemented+ assert ro.__eq__([]) is NotImplemented++ def test_unhashable(self):+ """``__hash__ = None`` prevents the equality/hash contract from being+ violated if anyone wraps the allow-list in a ``set``/``frozenset``."""+ ro = RegexOrigin(r"^x$")+ with pytest.raises(TypeError):+ hash(ro)+ with pytest.raises(TypeError):+ # `len({ro})` builds the set (calling __hash__) and consumes it via+ # an explicit function call — keeps ruff from collapsing the+ # statement and Sonar from flagging it as side-effect-free.+ len({ro})++ def test_no_redos(self):+ """Pattern must complete on hostile input — ``[^/]+`` has no nested+ quantifiers so backtracking is bounded. Threshold is generous (500ms)+ so noisy CI runners don't flake; ReDoS would blow up by orders of+ magnitude past this."""+ ro = RegexOrigin(r"^https://[^/]+\.example\.com$")+ hostile = "https://" + "a" * 10000 + ".evil.com"+ start = time.perf_counter()+ _ = hostile in [ro]+ assert time.perf_counter() - start < 0.5+++class TestNormalizeWebAppOrigin:+ def test_basic_https(self):+ origin, wildcard, _ = normalize_web_app_origin("https://example.com")+ assert origin == "https://example.com"+ assert wildcard == "https://*.example.com"++ def test_strips_trailing_slash(self):+ origin, _, _ = normalize_web_app_origin("https://example.com/")+ assert origin == "https://example.com"++ def test_strips_path_and_query(self):+ origin, _, _ = normalize_web_app_origin("https://example.com/path?q=1")+ assert origin == "https://example.com"++ def test_lowercases_hostname(self):+ """Browsers serialize ``Origin`` with a lowercase host (RFC 6454);+ django-cors-headers does case-sensitive string compare, so the env+ value must be lowercased to match."""+ origin, _, _ = normalize_web_app_origin("https://APP.EXAMPLE.COM")+ assert origin == "https://app.example.com"++ def test_drops_default_https_port(self):+ """Browsers omit the explicit default port from ``Origin`` per+ RFC 6454 — keeping ``:443`` would silently break exact match."""+ origin, _, _ = normalize_web_app_origin("https://example.com:443")+ assert origin == "https://example.com"++ def test_drops_default_http_port(self):+ # NOSONAR — `http://` URLs are intentional test data for the port+ # normalization logic, not a runtime use of the insecure protocol.+ origin, _, _ = normalize_web_app_origin("http://example.com:80") # NOSONAR+ assert origin == "http://example.com" # NOSONAR++ def test_keeps_non_default_port(self):+ origin, wildcard, _ = normalize_web_app_origin("https://example.com:8443")+ assert origin == "https://example.com:8443"+ assert wildcard == "https://*.example.com:8443"++ def test_localhost_default(self):+ origin, _, _ = normalize_web_app_origin("http://localhost:3000")+ assert origin == "http://localhost:3000"++ @pytest.mark.parametrize(+ "bad",+ [+ "",+ "not-a-url",+ "example.com", # missing scheme+ "//example.com", # protocol-relative+ "https://", # missing host+ "ftp://example.com", # non-browser scheme # NOSONAR — test input asserting ftp is rejected+ "ws://example.com", # not a top-level browser scheme+ "https://example.com:abc", # malformed port — urlparse raises on .port access+ "https://example.com:99999", # out-of-range port+ ],+ )+ def test_rejects_misconfigured(self, bad):+ """Fail fast at startup so misconfigured envs can't silently produce+ CORS rules that match nothing real."""+ with pytest.raises(ValueError, match="WEB_APP_ORIGIN_URL"):+ normalize_web_app_origin(bad)+++class TestSubdomainRegexEndToEnd:+ """End-to-end: the regex returned by ``normalize_web_app_origin`` is what+ actually gates production. Verify via ``RegexOrigin`` (same path as+ SocketIO uses)."""++ def test_apex_env_accepts_subdomains(self):+ _, _, pattern = normalize_web_app_origin("https://us-central.unstract.com")+ ro = RegexOrigin(pattern)+ # The exact failing origins from UN-3439:+ assert "https://dev.env.us-central.unstract.com" in [ro]+ assert "https://test.env.us-central.unstract.com" in [ro]++ def test_apex_rejected_by_regex(self):+ """Apex itself is *not* matched by the wildcard regex — that's the+ exact-match CORS_ALLOWED_ORIGINS entry's job."""+ _, _, pattern = normalize_web_app_origin("https://example.com")+ ro = RegexOrigin(pattern)+ assert "https://example.com" not in [ro]++ @pytest.mark.parametrize(+ "spoof",+ [+ "https://attacker-example.com",+ "https://x.example.com.attacker.com",+ "https://example.com.attacker.com",+ "http://app.example.com", # wrong scheme # NOSONAR — test input asserting http is rejected+ "https://app.example.com\n", # trailing newline+ ],+ )+ def test_spoofed_origins_rejected(self, spoof):+ _, _, pattern = normalize_web_app_origin("https://example.com")+ ro = RegexOrigin(pattern)+ assert spoof not in [ro], f"should reject {spoof!r}"++ def test_uppercase_env_still_accepts_lowercase_origin(self):+ """Browser sends lowercase even if the env was set with uppercase;+ normalization must canonicalize before regex compilation."""+ _, _, pattern = normalize_web_app_origin("https://APP.EXAMPLE.COM")+ ro = RegexOrigin(pattern)+ assert "https://sub.app.example.com" in [ro]