Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(security): keep absolute_uri and csv export requests on site_url

Matt Puab75010ee2cd5
diff --git a/posthog/test/test_utils.py b/posthog/test/test_utils.py
index f0a4fa934413..fd96f34a2a36 100644
--- a/posthog/test/test_utils.py
+++ b/posthog/test/test_utils.py
@@ -124,6 +124,12 @@ class TestAbsoluteUrls(SimpleTestCase):
"https://app.posthog.com",
"https://app.posthog.com/signup?email=someone%40example.com",
),
+ # Leading slashes are dropped, so a doubled slash stays a path on SITE_URL.
+ (
+ "//api/path",
+ "https://app.posthog.com",
+ "https://app.posthog.com/api/path",
+ ),
("/api/path", "", "/api/path"), # current behavior whether correct or not
(
"/api/path",
@@ -166,6 +172,15 @@ class TestAbsoluteUrls(SimpleTestCase):
("percent_encoded_slash", "https://attacker.example%[email protected]/path"),
("percent_encoded_question_mark", "https://attacker.example%[email protected]/path"),
("percent_encoded_hash", "https://attacker.example%[email protected]/path"),
+ # URL parsers strip these before reading the authority, so the value would
+ # resolve to another host while looking scheme-less here.
+ ("leading_tab", "\t//attacker.example/path"),
+ ("leading_newline", "\n//attacker.example/path"),
+ ("leading_space", " //attacker.example/path"),
+ ("leading_nul", "\x00//attacker.example/path"),
+ ("tab_between_slashes", "/\t//attacker.example/path"),
+ ("same_host_without_scheme", "\t//app.posthog.com/path"),
+ ("space_between_slashes", "/ //attacker.example/path"),
]
)
def test_absolute_uri_rejects_ambiguous_authority(self, _name: str, url: str) -> None:
diff --git a/posthog/utils.py b/posthog/utils.py
index 92c96daf3e09..7d4ea7015c51 100644
--- a/posthog/utils.py
+++ b/posthog/utils.py
@@ -108,6 +108,10 @@ logger = structlog.get_logger(__name__)
__location__ = os.path.realpath(os.path.join(os.getcwd(), os.path.dirname(__file__)))
+# C0 controls anywhere, and a leading space: characters URL parsers strip or drop before reading the authority.
+_URL_CONTROL_CHARS = re.compile(r"^[\x00-\x20]|[\x00-\x1f]")
+
+
class PotentialSecurityProblemException(Exception):
"""
When providing an absolutely-formatted URL
@@ -131,18 +135,29 @@ def absolute_uri(url: Optional[str] = None) -> str:
if has_ambiguous_authority(url):
raise PotentialSecurityProblemException(f"It is forbidden to provide an absolute URI using {url}")
+ # urlparse and urljoin drop tabs and newlines and strip leading control characters and
+ # spaces, so the value they resolve can differ from the one checked below.
+ if _URL_CONTROL_CHARS.search(url):
+ raise PotentialSecurityProblemException(f"It is forbidden to provide an absolute URI using {url}")
+
provided_url = urlparse(url)
if provided_url.hostname and provided_url.scheme:
site_url = urlparse(settings.SITE_URL)
- provided_url = provided_url
if (
site_url.hostname != provided_url.hostname
or site_url.port != provided_url.port
or site_url.scheme != provided_url.scheme
):
raise PotentialSecurityProblemException(f"It is forbidden to provide an absolute URI using {url}")
+ return urljoin(settings.SITE_URL.rstrip("/") + "/", url.lstrip("/"))
+
+ relative_url = url.lstrip("/")
+ # A relative value must stay on SITE_URL: anything still carrying an authority after the
+ # slashes are stripped would be read by urljoin as a protocol-relative URL.
+ if urlparse(relative_url).netloc:
+ raise PotentialSecurityProblemException(f"It is forbidden to provide an absolute URI using {url}")
- return urljoin(settings.SITE_URL.rstrip("/") + "/", url.lstrip("/"))
+ return urljoin(settings.SITE_URL.rstrip("/") + "/", relative_url)
@dataclasses.dataclass(frozen=True, kw_only=True, slots=True)
diff --git a/products/exports/backend/tasks/csv_exporter.py b/products/exports/backend/tasks/csv_exporter.py
index cd87f9fe6f2b..0e4a386a5b72 100644
--- a/products/exports/backend/tasks/csv_exporter.py
+++ b/products/exports/backend/tasks/csv_exporter.py
@@ -1,4 +1,5 @@
import os
+import re
import csv
import json
import datetime
@@ -10,8 +11,9 @@ from contextlib import contextmanager
from copy import deepcopy
from dataclasses import dataclass
from typing import Any, NoReturn, Optional, Protocol
-from urllib.parse import parse_qsl, quote, urlencode, urlparse, urlunparse
+from urllib.parse import parse_qsl, quote, unquote, urlencode, urlparse, urlunparse
+from django.conf import settings
from django.http import QueryDict
import requests
@@ -570,10 +572,21 @@ def get_from_insights_api(exported_asset: ExportedAsset, limit: int, resource: d
datetime.timedelta(minutes=15),
token_audience,
)
+ team_id = exported_asset.team_id
+ project_id = exported_asset.team.project_id
total = 0
while total < CSV_EXPORT_LIMIT:
try:
- response = make_api_call(access_token, body, limit, method, next_url, path)
+ response = make_api_call(
+ access_token,
+ body,
+ limit,
+ method,
+ next_url,
+ path,
+ team_id=team_id,
+ project_id=project_id,
+ )
except HTTPError as e:
# The underlying resource (e.g. a cohort) can be deleted or become unresolvable
# mid-export, which surfaces as a 404 partway through pagination. Treat that as
@@ -829,6 +842,37 @@ def get_limit_param_key(path: str) -> str:
return "breakdown_limit" if breakdown is not None else "limit"
+# Route prefixes the product sends for path-based exports, and the pagination links they return.
+# A prefix match: any GET under the export's own team is allowed, not a fixed set of endpoints.
+_TEAM_SCOPED_EXPORT_PATH = re.compile(r"^/api/(?P<kind>projects|environments)/(?P<id>[0-9]+)/")
+# The legacy cohort route carries no team id; the endpoint resolves the team itself.
+_UNSCOPED_EXPORT_PATHS = (re.compile(r"^/api/cohort/[0-9]+/persons/?$"),)
+
+
+def is_allowed_export_url(request_url: str, team_id: int, project_id: int) -> bool:
+ parsed = urlparse(request_url)
+ site_url = urlparse(settings.SITE_URL)
+ # Compared the way absolute_uri compares them, so host casing in a pagination link doesn't matter.
+ if (parsed.scheme, parsed.hostname, parsed.port) != (site_url.scheme, site_url.hostname, site_url.port):
+ return False
+
+ path = parsed.path
+ if any(segment in (".", "..") for segment in unquote(path).split("/")):
+ return False
+
+ site_path = site_url.path.rstrip("/")
+ if site_path:
+ if not path.startswith(f"{site_path}/"):
+ return False
+ path = path.removeprefix(site_path)
+
+ team_scoped = _TEAM_SCOPED_EXPORT_PATH.match(path)
+ if team_scoped:
+ allowed_ids = (team_id,) if team_scoped["kind"] == "environments" else (team_id, project_id)
+ return int(team_scoped["id"]) in allowed_ids
+ return any(pattern.match(path) for pattern in _UNSCOPED_EXPORT_PATHS)
+
+
def make_api_call(
access_token: str,
body: Any,
@@ -836,8 +880,18 @@ def make_api_call(
method: str,
next_url: Optional[str],
path: str,
+ team_id: int,
+ project_id: int,
) -> requests.models.Response:
request_url: str = absolute_uri(next_url or path)
+ if not is_allowed_export_url(request_url, team_id, project_id):
+ logger.warning(
+ "csv_exporter.path_not_allowlisted",
+ team_id=team_id,
+ request_path=urlparse(request_url).path,
+ is_next_url=next_url is not None,
+ )
+ raise ValueError("This export request is no longer supported. Create a new export and try again.")
url = add_query_params(
request_url,
{get_limit_param_key(request_url): str(limit), "is_csv_export": "1"},
@@ -848,7 +902,17 @@ def make_api_call(
json=body,
headers={"Authorization": f"Bearer {access_token}"},
timeout=60,
+ # The allowlist checked only this URL, and a redirect would take the request elsewhere.
+ allow_redirects=False,
)
+ if 300 <= response.status_code < 400:
+ logger.warning(
+ "csv_exporter.redirect_refused",
+ team_id=team_id,
+ request_path=urlparse(request_url).path,
+ status_code=response.status_code,
+ )
+ raise ValueError("This export request is no longer supported. Create a new export and try again.")
response.raise_for_status()
return response
diff --git a/products/exports/backend/tasks/test/test_csv_exporter.py b/products/exports/backend/tasks/test/test_csv_exporter.py
index af346219eeba..28cef2b2a921 100644
--- a/products/exports/backend/tasks/test/test_csv_exporter.py
+++ b/products/exports/backend/tasks/test/test_csv_exporter.py
@@ -7,7 +7,6 @@ from typing import Any, Optional
import pytest
import time_machine
from posthog.test.base import APIBaseTest, _create_event, _create_person, flush_persons_and_events
-from unittest import mock
from unittest.mock import ANY, MagicMock, Mock, patch
from django.test import override_settings
@@ -69,7 +68,7 @@ class TestCSVExporter(APIBaseTest):
# API responses copied from https://github.com/PostHog/posthog/runs/7221634689?check_suite_focus=true
mock_response.json.side_effect = [
{
- "next": "http://testserver/api/projects/169/events?orderBy=%5B%22-timestamp%22%5D&properties=%5B%7B%22key%22%3A%22%24browser%22%2C%22value%22%3A%5B%22Safari%22%5D%2C%22operator%22%3A%22exact%22%2C%22type%22%3A%22event%22%7D%5D&after=2022-07-06T19%3A27%3A43.206326&limit=1&before=2022-07-06T19%3A37%3A43.095295%2B00%3A00",
+ "next": f"http://testserver/api/projects/{self.team.id}/events?orderBy=%5B%22-timestamp%22%5D&properties=%5B%7B%22key%22%3A%22%24browser%22%2C%22value%22%3A%5B%22Safari%22%5D%2C%22operator%22%3A%22exact%22%2C%22type%22%3A%22event%22%7D%5D&after=2022-07-06T19%3A27%3A43.206326&limit=1&before=2022-07-06T19%3A37%3A43.095295%2B00%3A00",
"results": [
{
"id": "e9ca132e-400f-4854-a83c-16c151b2f145",
@@ -84,7 +83,7 @@ class TestCSVExporter(APIBaseTest):
],
},
{
- "next": "http://testserver/api/projects/169/events?orderBy=%5B%22-timestamp%22%5D&properties=%5B%7B%22key%22%3A%22%24browser%22%2C%22value%22%3A%5B%22Safari%22%5D%2C%22operator%22%3A%22exact%22%2C%22type%22%3A%22event%22%7D%5D&after=2022-07-06T19%3A27%3A43.206326&limit=1&before=2022-07-06T19%3A37%3A43.095279%2B00%3A00",
+ "next": f"http://testserver/api/projects/{self.team.id}/events?orderBy=%5B%22-timestamp%22%5D&properties=%5B%7B%22key%22%3A%22%24browser%22%2C%22value%22%3A%5B%22Safari%22%5D%2C%22operator%22%3A%22exact%22%2C%22type%22%3A%22event%22%7D%5D&after=2022-07-06T19%3A27%3A43.206326&limit=1&before=2022-07-06T19%3A37%3A43.095279%2B00%3A00",
"results": [
{
"id": "1624228e-a4f1-48cd-aabc-6baa3ddb22e4",
@@ -124,7 +123,7 @@ class TestCSVExporter(APIBaseTest):
team=self.team,
created_by=self.user,
export_format=ExportedAsset.ExportFormat.CSV,
- export_context={"path": "/api/literally/anything", **extra_context},
+ export_context={"path": f"/api/environments/{self.team.id}/persons/", **extra_context},
source_authentication=ExportedAsset.SourceAuthentication.SESSION,
)
asset.save()
@@ -322,7 +321,7 @@ class TestCSVExporter(APIBaseTest):
def test_csv_exporter_limits_breakdown_insights_correctly(
self, mocked_request: Any, mocked_object_storage_write: Any, mocked_uuidt: Any
) -> None:
- path = "api/projects/1/insights/trend/?insight=TRENDS&breakdown=email&date_from=-7d"
+ path = f"api/projects/{self.team.id}/insights/trend/?insight=TRENDS&breakdown=email&date_from=-7d"
exported_asset = self._create_asset({"path": path})
mock_response = Mock()
mock_response.status_code = 200
@@ -337,6 +336,7 @@ class TestCSVExporter(APIBaseTest):
timeout=60,
json=None,
headers=ANY,
+ allow_redirects=False,
)
@patch("products.exports.backend.tasks.csv_exporter.logger")
@@ -446,7 +446,7 @@ class TestCSVExporter(APIBaseTest):
csv_exporter.export_tabular(exported_asset)
assert patched_make_api_call.call_count == 4
- patched_make_api_call.assert_called_with(mock.ANY, mock.ANY, 64, mock.ANY, mock.ANY, mock.ANY)
+ assert patched_make_api_call.call_args.args[2] == 64
@patch("products.exports.backend.tasks.csv_exporter.logger")
def test_404_mid_export_returns_rows_already_fetched(self, _mock_logger: MagicMock) -> None:
@@ -1751,7 +1751,7 @@ class TestCSVExporter(APIBaseTest):
exported_asset = ExportedAsset(
team=self.team,
export_format=ExportedAsset.ExportFormat.XLSX,
- export_context={"path": "/api/test/endpoint"},
+ export_context={"path": f"/api/environments/{self.team.id}/persons/"},
source_authentication=ExportedAsset.SourceAuthentication.SESSION,
)
exported_asset.save()
diff --git a/products/exports/backend/tasks/test/test_csv_exporter_renders.py b/products/exports/backend/tasks/test/test_csv_exporter_renders.py
index 30d722cba5c3..aace5901cfbc 100644
--- a/products/exports/backend/tasks/test/test_csv_exporter_renders.py
+++ b/products/exports/backend/tasks/test/test_csv_exporter_renders.py
@@ -41,7 +41,7 @@ def test_csv_rendering(
asset = ExportedAsset(
team=team,
export_format=ExportedAsset.ExportFormat.CSV,
- export_context={"path": "/api/literally/anything"},
+ export_context={"path": f"/api/environments/{team.id}/persons/"},
source_authentication=ExportedAsset.SourceAuthentication.SESSION,
)
export_context = cast(dict[str, Any], asset.export_context)
diff --git a/products/exports/backend/tasks/test/test_csv_exporter_url_sanitising.py b/products/exports/backend/tasks/test/test_csv_exporter_url_sanitising.py
index 189951f4a61b..4e9e18fb1861 100644
--- a/products/exports/backend/tasks/test/test_csv_exporter_url_sanitising.py
+++ b/products/exports/backend/tasks/test/test_csv_exporter_url_sanitising.py
@@ -1,9 +1,15 @@
+import pytest
+from unittest.mock import MagicMock, patch
+
from django.test import SimpleTestCase
+import responses
from parameterized import parameterized
from posthog.utils import PotentialSecurityProblemException, absolute_uri
+from products.exports.backend.tasks.csv_exporter import is_allowed_export_url, make_api_call
+
class TestCSVExporterURLSanitization(SimpleTestCase):
def test_sanitize_url_when_provided_path(self) -> None:
@@ -48,3 +54,61 @@ class TestCSVExporterURLSanitization(SimpleTestCase):
def test_sanitise_url_error_cases_as_next_url(self, _name: str, site_url: str, provided_url_or_path: str) -> None:
with self.settings(SITE_URL=site_url), self.assertRaises(PotentialSecurityProblemException):
absolute_uri(provided_url_or_path or None)
+
+
+class TestExportURLAllowlist(SimpleTestCase):
+ @parameterized.expand(
+ [
+ ("environment_persons", "https://app.posthog.com/api/environments/2/persons?limit=100", True),
+ ("project_by_team_id", "https://app.posthog.com/api/projects/2/advanced_activity_logs/?page=2", True),
+ ("project_by_project_id", "https://app.posthog.com/api/projects/7/llm_analytics/trace_reviews/", True),
+ ("cohort_persons", "https://app.posthog.com/api/cohort/5/persons?offset=100", True),
+ ("cohort_persons_trailing_slash", "https://app.posthog.com/api/cohort/5/persons/", True),
+ ("mixed_case_host", "https://App.PostHog.com/api/environments/2/persons", True),
+ ("environment_by_project_id", "https://app.posthog.com/api/environments/7/persons", False),
+ ("non_ascii_digit", "https://app.posthog.com/api/environments/\u0662/persons", False),
+ ("other_team", "https://app.posthog.com/api/environments/3/persons", False),
+ ("other_project", "https://app.posthog.com/api/projects/8/insights/", False),
+ ("unlisted_api_route", "https://app.posthog.com/api/users/@me/", False),
+ ("non_api_path", "https://app.posthog.com/admin/", False),
+ ("other_cohort_route", "https://app.posthog.com/api/cohort/5/", False),
+ ("other_host", "https://other.example/api/environments/2/persons", False),
+ ("other_scheme", "http://app.posthog.com/api/environments/2/persons", False),
+ ("dot_segments", "https://app.posthog.com/api/environments/2/../../users/@me/", False),
+ ("encoded_dot_segments", "https://app.posthog.com/api/environments/2/%2e%2e/%2e%2e/users/", False),
+ ]
+ )
+ def test_is_allowed_export_url(self, _name: str, url: str, expected: bool) -> None:
+ with self.settings(SITE_URL="https://app.posthog.com"):
+ assert is_allowed_export_url(url, team_id=2, project_id=7) is expected
+
+ @patch("products.exports.backend.tasks.csv_exporter.requests.request")
+ def test_make_api_call_outside_allowlist(self, patched_request: MagicMock) -> None:
+ with self.settings(SITE_URL="https://app.posthog.com"):
+ with pytest.raises(ValueError, match="no longer supported"):
+ make_api_call("token", None, 100, "GET", None, "/api/users/@me/", team_id=2, project_id=7)
+
+ assert not patched_request.called
+
+ @responses.activate
+ def test_make_api_call_refuses_redirect_off_allowlisted_path(self) -> None:
+ allowed_url = "https://app.posthog.com/api/environments/2/persons"
+ responses.get(allowed_url, status=302, headers={"Location": "https://attacker.example/x"})
+ attacker = responses.get("https://attacker.example/x", json={"results": [{"secret": 1}]})
+
+ with self.settings(SITE_URL="https://app.posthog.com"):
+ with pytest.raises(ValueError, match="no longer supported"):
+ make_api_call("token", None, 100, "GET", None, "/api/environments/2/persons", team_id=2, project_id=7)
+
+ assert attacker.call_count == 0
+
+ @parameterized.expand(
+ [
+ ("under_site_path", "https://example.com/posthog/api/environments/2/persons", True),
+ ("outside_site_path", "https://example.com/api/environments/2/persons", False),
+ ("sibling_of_site_path", "https://example.com/posthog-other/api/environments/2/persons", False),
+ ]
+ )
+ def test_is_allowed_export_url_with_site_path(self, _name: str, url: str, expected: bool) -> None:
+ with self.settings(SITE_URL="https://example.com/posthog"):
+ assert is_allowed_export_url(url, team_id=2, project_id=7) is expected