:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(security): keep absolute_uri and csv export requests on site_url
Matt Puab75010ee2cd5
diff --git a/posthog/test/test_utils.py b/posthog/test/test_utils.pyindex f0a4fa934413..fd96f34a2a36 100644--- a/posthog/test/test_utils.py+++ b/posthog/test/test_utils.py@@ -124,6 +124,12 @@ class TestAbsoluteUrls(SimpleTestCase):"https://app.posthog.com","https://app.posthog.com/signup?email=someone%40example.com",),+ # Leading slashes are dropped, so a doubled slash stays a path on SITE_URL.+ (+ "//api/path",+ "https://app.posthog.com",+ "https://app.posthog.com/api/path",+ ),("/api/path", "", "/api/path"), # current behavior whether correct or not("/api/path",@@ -166,6 +172,15 @@ class TestAbsoluteUrls(SimpleTestCase):("percent_encoded_slash", "https://attacker.example%[email protected]/path"),("percent_encoded_question_mark", "https://attacker.example%[email protected]/path"),("percent_encoded_hash", "https://attacker.example%[email protected]/path"),+ # URL parsers strip these before reading the authority, so the value would+ # resolve to another host while looking scheme-less here.+ ("leading_tab", "\t//attacker.example/path"),+ ("leading_newline", "\n//attacker.example/path"),+ ("leading_space", " //attacker.example/path"),+ ("leading_nul", "\x00//attacker.example/path"),+ ("tab_between_slashes", "/\t//attacker.example/path"),+ ("same_host_without_scheme", "\t//app.posthog.com/path"),+ ("space_between_slashes", "/ //attacker.example/path"),])def test_absolute_uri_rejects_ambiguous_authority(self, _name: str, url: str) -> None:diff --git a/posthog/utils.py b/posthog/utils.pyindex 92c96daf3e09..7d4ea7015c51 100644--- a/posthog/utils.py+++ b/posthog/utils.py@@ -108,6 +108,10 @@ logger = structlog.get_logger(__name__)__location__ = os.path.realpath(os.path.join(os.getcwd(), os.path.dirname(__file__)))+# C0 controls anywhere, and a leading space: characters URL parsers strip or drop before reading the authority.+_URL_CONTROL_CHARS = re.compile(r"^[\x00-\x20]|[\x00-\x1f]")++class PotentialSecurityProblemException(Exception):"""When providing an absolutely-formatted URL@@ -131,18 +135,29 @@ def absolute_uri(url: Optional[str] = None) -> str:if has_ambiguous_authority(url):raise PotentialSecurityProblemException(f"It is forbidden to provide an absolute URI using {url}")+ # urlparse and urljoin drop tabs and newlines and strip leading control characters and+ # spaces, so the value they resolve can differ from the one checked below.+ if _URL_CONTROL_CHARS.search(url):+ raise PotentialSecurityProblemException(f"It is forbidden to provide an absolute URI using {url}")+provided_url = urlparse(url)if provided_url.hostname and provided_url.scheme:site_url = urlparse(settings.SITE_URL)- provided_url = provided_urlif (site_url.hostname != provided_url.hostnameor site_url.port != provided_url.portor site_url.scheme != provided_url.scheme):raise PotentialSecurityProblemException(f"It is forbidden to provide an absolute URI using {url}")+ return urljoin(settings.SITE_URL.rstrip("/") + "/", url.lstrip("/"))++ relative_url = url.lstrip("/")+ # A relative value must stay on SITE_URL: anything still carrying an authority after the+ # slashes are stripped would be read by urljoin as a protocol-relative URL.+ if urlparse(relative_url).netloc:+ raise PotentialSecurityProblemException(f"It is forbidden to provide an absolute URI using {url}")- return urljoin(settings.SITE_URL.rstrip("/") + "/", url.lstrip("/"))+ return urljoin(settings.SITE_URL.rstrip("/") + "/", relative_url)@dataclasses.dataclass(frozen=True, kw_only=True, slots=True)diff --git a/products/exports/backend/tasks/csv_exporter.py b/products/exports/backend/tasks/csv_exporter.pyindex cd87f9fe6f2b..0e4a386a5b72 100644--- a/products/exports/backend/tasks/csv_exporter.py+++ b/products/exports/backend/tasks/csv_exporter.py@@ -1,4 +1,5 @@import os+import reimport csvimport jsonimport datetime@@ -10,8 +11,9 @@ from contextlib import contextmanagerfrom copy import deepcopyfrom dataclasses import dataclassfrom typing import Any, NoReturn, Optional, Protocol-from urllib.parse import parse_qsl, quote, urlencode, urlparse, urlunparse+from urllib.parse import parse_qsl, quote, unquote, urlencode, urlparse, urlunparse+from django.conf import settingsfrom django.http import QueryDictimport requests@@ -570,10 +572,21 @@ def get_from_insights_api(exported_asset: ExportedAsset, limit: int, resource: ddatetime.timedelta(minutes=15),token_audience,)+ team_id = exported_asset.team_id+ project_id = exported_asset.team.project_idtotal = 0while total < CSV_EXPORT_LIMIT:try:- response = make_api_call(access_token, body, limit, method, next_url, path)+ response = make_api_call(+ access_token,+ body,+ limit,+ method,+ next_url,+ path,+ team_id=team_id,+ project_id=project_id,+ )except HTTPError as e:# The underlying resource (e.g. a cohort) can be deleted or become unresolvable# mid-export, which surfaces as a 404 partway through pagination. Treat that as@@ -829,6 +842,37 @@ def get_limit_param_key(path: str) -> str:return "breakdown_limit" if breakdown is not None else "limit"+# Route prefixes the product sends for path-based exports, and the pagination links they return.+# A prefix match: any GET under the export's own team is allowed, not a fixed set of endpoints.+_TEAM_SCOPED_EXPORT_PATH = re.compile(r"^/api/(?P<kind>projects|environments)/(?P<id>[0-9]+)/")+# The legacy cohort route carries no team id; the endpoint resolves the team itself.+_UNSCOPED_EXPORT_PATHS = (re.compile(r"^/api/cohort/[0-9]+/persons/?$"),)+++def is_allowed_export_url(request_url: str, team_id: int, project_id: int) -> bool:+ parsed = urlparse(request_url)+ site_url = urlparse(settings.SITE_URL)+ # Compared the way absolute_uri compares them, so host casing in a pagination link doesn't matter.+ if (parsed.scheme, parsed.hostname, parsed.port) != (site_url.scheme, site_url.hostname, site_url.port):+ return False++ path = parsed.path+ if any(segment in (".", "..") for segment in unquote(path).split("/")):+ return False++ site_path = site_url.path.rstrip("/")+ if site_path:+ if not path.startswith(f"{site_path}/"):+ return False+ path = path.removeprefix(site_path)++ team_scoped = _TEAM_SCOPED_EXPORT_PATH.match(path)+ if team_scoped:+ allowed_ids = (team_id,) if team_scoped["kind"] == "environments" else (team_id, project_id)+ return int(team_scoped["id"]) in allowed_ids+ return any(pattern.match(path) for pattern in _UNSCOPED_EXPORT_PATHS)++def make_api_call(access_token: str,body: Any,@@ -836,8 +880,18 @@ def make_api_call(method: str,next_url: Optional[str],path: str,+ team_id: int,+ project_id: int,) -> requests.models.Response:request_url: str = absolute_uri(next_url or path)+ if not is_allowed_export_url(request_url, team_id, project_id):+ logger.warning(+ "csv_exporter.path_not_allowlisted",+ team_id=team_id,+ request_path=urlparse(request_url).path,+ is_next_url=next_url is not None,+ )+ raise ValueError("This export request is no longer supported. Create a new export and try again.")url = add_query_params(request_url,{get_limit_param_key(request_url): str(limit), "is_csv_export": "1"},@@ -848,7 +902,17 @@ def make_api_call(json=body,headers={"Authorization": f"Bearer {access_token}"},timeout=60,+ # The allowlist checked only this URL, and a redirect would take the request elsewhere.+ allow_redirects=False,)+ if 300 <= response.status_code < 400:+ logger.warning(+ "csv_exporter.redirect_refused",+ team_id=team_id,+ request_path=urlparse(request_url).path,+ status_code=response.status_code,+ )+ raise ValueError("This export request is no longer supported. Create a new export and try again.")response.raise_for_status()return responsediff --git a/products/exports/backend/tasks/test/test_csv_exporter.py b/products/exports/backend/tasks/test/test_csv_exporter.pyindex af346219eeba..28cef2b2a921 100644--- a/products/exports/backend/tasks/test/test_csv_exporter.py+++ b/products/exports/backend/tasks/test/test_csv_exporter.py@@ -7,7 +7,6 @@ from typing import Any, Optionalimport pytestimport time_machinefrom posthog.test.base import APIBaseTest, _create_event, _create_person, flush_persons_and_events-from unittest import mockfrom unittest.mock import ANY, MagicMock, Mock, patchfrom django.test import override_settings@@ -69,7 +68,7 @@ class TestCSVExporter(APIBaseTest):# API responses copied from https://github.com/PostHog/posthog/runs/7221634689?check_suite_focus=truemock_response.json.side_effect = [{- "next": "http://testserver/api/projects/169/events?orderBy=%5B%22-timestamp%22%5D&properties=%5B%7B%22key%22%3A%22%24browser%22%2C%22value%22%3A%5B%22Safari%22%5D%2C%22operator%22%3A%22exact%22%2C%22type%22%3A%22event%22%7D%5D&after=2022-07-06T19%3A27%3A43.206326&limit=1&before=2022-07-06T19%3A37%3A43.095295%2B00%3A00",+ "next": f"http://testserver/api/projects/{self.team.id}/events?orderBy=%5B%22-timestamp%22%5D&properties=%5B%7B%22key%22%3A%22%24browser%22%2C%22value%22%3A%5B%22Safari%22%5D%2C%22operator%22%3A%22exact%22%2C%22type%22%3A%22event%22%7D%5D&after=2022-07-06T19%3A27%3A43.206326&limit=1&before=2022-07-06T19%3A37%3A43.095295%2B00%3A00","results": [{"id": "e9ca132e-400f-4854-a83c-16c151b2f145",@@ -84,7 +83,7 @@ class TestCSVExporter(APIBaseTest):],},{- "next": "http://testserver/api/projects/169/events?orderBy=%5B%22-timestamp%22%5D&properties=%5B%7B%22key%22%3A%22%24browser%22%2C%22value%22%3A%5B%22Safari%22%5D%2C%22operator%22%3A%22exact%22%2C%22type%22%3A%22event%22%7D%5D&after=2022-07-06T19%3A27%3A43.206326&limit=1&before=2022-07-06T19%3A37%3A43.095279%2B00%3A00",+ "next": f"http://testserver/api/projects/{self.team.id}/events?orderBy=%5B%22-timestamp%22%5D&properties=%5B%7B%22key%22%3A%22%24browser%22%2C%22value%22%3A%5B%22Safari%22%5D%2C%22operator%22%3A%22exact%22%2C%22type%22%3A%22event%22%7D%5D&after=2022-07-06T19%3A27%3A43.206326&limit=1&before=2022-07-06T19%3A37%3A43.095279%2B00%3A00","results": [{"id": "1624228e-a4f1-48cd-aabc-6baa3ddb22e4",@@ -124,7 +123,7 @@ class TestCSVExporter(APIBaseTest):team=self.team,created_by=self.user,export_format=ExportedAsset.ExportFormat.CSV,- export_context={"path": "/api/literally/anything", **extra_context},+ export_context={"path": f"/api/environments/{self.team.id}/persons/", **extra_context},source_authentication=ExportedAsset.SourceAuthentication.SESSION,)asset.save()@@ -322,7 +321,7 @@ class TestCSVExporter(APIBaseTest):def test_csv_exporter_limits_breakdown_insights_correctly(self, mocked_request: Any, mocked_object_storage_write: Any, mocked_uuidt: Any) -> None:- path = "api/projects/1/insights/trend/?insight=TRENDS&breakdown=email&date_from=-7d"+ path = f"api/projects/{self.team.id}/insights/trend/?insight=TRENDS&breakdown=email&date_from=-7d"exported_asset = self._create_asset({"path": path})mock_response = Mock()mock_response.status_code = 200@@ -337,6 +336,7 @@ class TestCSVExporter(APIBaseTest):timeout=60,json=None,headers=ANY,+ allow_redirects=False,)@patch("products.exports.backend.tasks.csv_exporter.logger")@@ -446,7 +446,7 @@ class TestCSVExporter(APIBaseTest):csv_exporter.export_tabular(exported_asset)assert patched_make_api_call.call_count == 4- patched_make_api_call.assert_called_with(mock.ANY, mock.ANY, 64, mock.ANY, mock.ANY, mock.ANY)+ assert patched_make_api_call.call_args.args[2] == 64@patch("products.exports.backend.tasks.csv_exporter.logger")def test_404_mid_export_returns_rows_already_fetched(self, _mock_logger: MagicMock) -> None:@@ -1751,7 +1751,7 @@ class TestCSVExporter(APIBaseTest):exported_asset = ExportedAsset(team=self.team,export_format=ExportedAsset.ExportFormat.XLSX,- export_context={"path": "/api/test/endpoint"},+ export_context={"path": f"/api/environments/{self.team.id}/persons/"},source_authentication=ExportedAsset.SourceAuthentication.SESSION,)exported_asset.save()diff --git a/products/exports/backend/tasks/test/test_csv_exporter_renders.py b/products/exports/backend/tasks/test/test_csv_exporter_renders.pyindex 30d722cba5c3..aace5901cfbc 100644--- a/products/exports/backend/tasks/test/test_csv_exporter_renders.py+++ b/products/exports/backend/tasks/test/test_csv_exporter_renders.py@@ -41,7 +41,7 @@ def test_csv_rendering(asset = ExportedAsset(team=team,export_format=ExportedAsset.ExportFormat.CSV,- export_context={"path": "/api/literally/anything"},+ export_context={"path": f"/api/environments/{team.id}/persons/"},source_authentication=ExportedAsset.SourceAuthentication.SESSION,)export_context = cast(dict[str, Any], asset.export_context)diff --git a/products/exports/backend/tasks/test/test_csv_exporter_url_sanitising.py b/products/exports/backend/tasks/test/test_csv_exporter_url_sanitising.pyindex 189951f4a61b..4e9e18fb1861 100644--- a/products/exports/backend/tasks/test/test_csv_exporter_url_sanitising.py+++ b/products/exports/backend/tasks/test/test_csv_exporter_url_sanitising.py@@ -1,9 +1,15 @@+import pytest+from unittest.mock import MagicMock, patch+from django.test import SimpleTestCase+import responsesfrom parameterized import parameterizedfrom posthog.utils import PotentialSecurityProblemException, absolute_uri+from products.exports.backend.tasks.csv_exporter import is_allowed_export_url, make_api_call+class TestCSVExporterURLSanitization(SimpleTestCase):def test_sanitize_url_when_provided_path(self) -> None:@@ -48,3 +54,61 @@ class TestCSVExporterURLSanitization(SimpleTestCase):def test_sanitise_url_error_cases_as_next_url(self, _name: str, site_url: str, provided_url_or_path: str) -> None:with self.settings(SITE_URL=site_url), self.assertRaises(PotentialSecurityProblemException):absolute_uri(provided_url_or_path or None)+++class TestExportURLAllowlist(SimpleTestCase):+ @parameterized.expand(+ [+ ("environment_persons", "https://app.posthog.com/api/environments/2/persons?limit=100", True),+ ("project_by_team_id", "https://app.posthog.com/api/projects/2/advanced_activity_logs/?page=2", True),+ ("project_by_project_id", "https://app.posthog.com/api/projects/7/llm_analytics/trace_reviews/", True),+ ("cohort_persons", "https://app.posthog.com/api/cohort/5/persons?offset=100", True),+ ("cohort_persons_trailing_slash", "https://app.posthog.com/api/cohort/5/persons/", True),+ ("mixed_case_host", "https://App.PostHog.com/api/environments/2/persons", True),+ ("environment_by_project_id", "https://app.posthog.com/api/environments/7/persons", False),+ ("non_ascii_digit", "https://app.posthog.com/api/environments/\u0662/persons", False),+ ("other_team", "https://app.posthog.com/api/environments/3/persons", False),+ ("other_project", "https://app.posthog.com/api/projects/8/insights/", False),+ ("unlisted_api_route", "https://app.posthog.com/api/users/@me/", False),+ ("non_api_path", "https://app.posthog.com/admin/", False),+ ("other_cohort_route", "https://app.posthog.com/api/cohort/5/", False),+ ("other_host", "https://other.example/api/environments/2/persons", False),+ ("other_scheme", "http://app.posthog.com/api/environments/2/persons", False),+ ("dot_segments", "https://app.posthog.com/api/environments/2/../../users/@me/", False),+ ("encoded_dot_segments", "https://app.posthog.com/api/environments/2/%2e%2e/%2e%2e/users/", False),+ ]+ )+ def test_is_allowed_export_url(self, _name: str, url: str, expected: bool) -> None:+ with self.settings(SITE_URL="https://app.posthog.com"):+ assert is_allowed_export_url(url, team_id=2, project_id=7) is expected++ @patch("products.exports.backend.tasks.csv_exporter.requests.request")+ def test_make_api_call_outside_allowlist(self, patched_request: MagicMock) -> None:+ with self.settings(SITE_URL="https://app.posthog.com"):+ with pytest.raises(ValueError, match="no longer supported"):+ make_api_call("token", None, 100, "GET", None, "/api/users/@me/", team_id=2, project_id=7)++ assert not patched_request.called++ @responses.activate+ def test_make_api_call_refuses_redirect_off_allowlisted_path(self) -> None:+ allowed_url = "https://app.posthog.com/api/environments/2/persons"+ responses.get(allowed_url, status=302, headers={"Location": "https://attacker.example/x"})+ attacker = responses.get("https://attacker.example/x", json={"results": [{"secret": 1}]})++ with self.settings(SITE_URL="https://app.posthog.com"):+ with pytest.raises(ValueError, match="no longer supported"):+ make_api_call("token", None, 100, "GET", None, "/api/environments/2/persons", team_id=2, project_id=7)++ assert attacker.call_count == 0++ @parameterized.expand(+ [+ ("under_site_path", "https://example.com/posthog/api/environments/2/persons", True),+ ("outside_site_path", "https://example.com/api/environments/2/persons", False),+ ("sibling_of_site_path", "https://example.com/posthog-other/api/environments/2/persons", False),+ ]+ )+ def test_is_allowed_export_url_with_site_path(self, _name: str, url: str, expected: bool) -> None:+ with self.settings(SITE_URL="https://example.com/posthog"):+ assert is_allowed_export_url(url, team_id=2, project_id=7) is expected