:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(alerts-platform): run metrics alert queries as their own clickhouse user (#130477)
Jimmy Zhu6b9b56f50fc6
diff --git a/posthog/api/snuffle_proxy.py b/posthog/api/snuffle_proxy.pyindex 471dea4a75af..52d3d1f31e32 100644--- a/posthog/api/snuffle_proxy.py+++ b/posthog/api/snuffle_proxy.py@@ -20,6 +20,7 @@ from django.http import HttpResponse, QueryDictimport requestsimport structlogfrom drf_spectacular.utils import extend_schema+from requests.auth import HTTPBasicAuthfrom rest_framework import status, viewsetsfrom rest_framework.decorators import actionfrom rest_framework.parsers import FormParser@@ -27,6 +28,7 @@ from rest_framework.request import Requestfrom posthog.api.routing import TeamAndOrgViewSetMixinfrom posthog.api_queries_budget import debit+from posthog.clickhouse.client.connection import ClickHouseUser, Workload, connection_credsfrom posthog.hogql_queries.query_runner import api_queries_budget_enforcement_enabled, get_api_queries_budget_statusfrom posthog.permissions import PostHogFeatureFlagPermissionfrom posthog.rate_limit import ClickHouseBurstRateThrottle, ClickHouseSustainedRateThrottle@@ -118,6 +120,15 @@ class SnuffleNotConfiguredError(Exception):"""Raised when no Snuffle URL is configured for this deployment."""+def _snuffle_auth(ch_user: ClickHouseUser | None) -> HTTPBasicAuth | None:+ # A token-backed user's file rotates, so the credential is read for each request.+ if ch_user is not None and (creds := connection_creds(Workload.LOGS, ch_user)) is not None:+ return HTTPBasicAuth(creds.user, creds.read_password())+ return (+ HTTPBasicAuth(settings.SNUFFLE_APM_USER, settings.SNUFFLE_APM_PASSWORD) if settings.SNUFFLE_APM_USER else None+ )++def snuffle_request(team_id: int,method: str,@@ -127,17 +138,19 @@ def snuffle_request(data: dict[str, list[str]] | dict[str, str] | None = None,accept: str = "application/json",timeout: float | None = None,+ ch_user: ClickHouseUser | None = None,) -> requests.Response:"""Send one request to Snuffle as `team_id` and debit the bytes it read from the query budget.- `upstream_path` is the Snuffle path, for example ``/api/v1/query_range``. Network errors from+ `upstream_path` is the Snuffle path, for example ``/api/v1/query_range``. Snuffle connects to+ ClickHouse as `ch_user` when it is a registered logs cluster user. Network errors from`requests` propagate, so each caller can map them to its own error format."""base_url = settings.SNUFFLE_APM_URLif not base_url:raise SnuffleNotConfiguredError("PromQL/LogQL query API is not configured")headers = {TEAM_ID_HEADER: str(team_id), "Accept": accept}- auth = (settings.SNUFFLE_APM_USER, settings.SNUFFLE_APM_PASSWORD) if settings.SNUFFLE_APM_USER else None+ auth = _snuffle_auth(ch_user)upstream = internal_requests.request(method,f"{base_url.rstrip('/')}{upstream_path}",diff --git a/posthog/clickhouse/client/connection.py b/posthog/clickhouse/client/connection.pyindex fd641c59afc7..69f813b72686 100644--- a/posthog/clickhouse/client/connection.py+++ b/posthog/clickhouse/client/connection.py@@ -160,6 +160,8 @@ class ClickHouseUser(StrEnum):ALERTS_PLATFORM_INSIGHT = "alerts_platform_insight"# Exists on the logs cluster only. Pass it with Workload.LOGS, never as a query tag.ALERTS_PLATFORM_LOGS = "alerts_platform_logs"+ # Exists on the logs cluster only. Snuffle connects as it for the platform's PromQL queries.+ ALERTS_PLATFORM_METRICS = "alerts_platform_metrics"# The platform's own bookkeeping, shared by every source: its check history and the default# user of its evaluation worker. A source's checks keep their own user, so a new source never# takes from another source's budget.@@ -451,7 +453,9 @@ def get_http_client(**overrides):# Named users that exist on the logs cluster. Every other named user exists on the main cluster# only, and sync_execute assigns some of them (APP, API) from tags without checking the workload.-LOGS_CLUSTER_USERS: frozenset[ClickHouseUser] = frozenset({ClickHouseUser.ALERTS_PLATFORM_LOGS})+LOGS_CLUSTER_USERS: frozenset[ClickHouseUser] = frozenset(+ {ClickHouseUser.ALERTS_PLATFORM_LOGS, ClickHouseUser.ALERTS_PLATFORM_METRICS}+)def connection_creds(workload: Workload, ch_user: ClickHouseUser) -> ClickHouseCredentials | None:diff --git a/products/logs/backend/test/test_loki_api.py b/products/logs/backend/test/test_loki_api.pyindex 1368c2b1c818..2e1ef1274d36 100644--- a/products/logs/backend/test/test_loki_api.py+++ b/products/logs/backend/test/test_loki_api.py@@ -5,6 +5,7 @@ from django.test import SimpleTestCase, override_settingsimport requestsfrom parameterized import parameterized+from requests.auth import HTTPBasicAuthfrom rest_framework import statusimport posthog.api.snuffle_proxy as snuffle_proxy@@ -67,7 +68,7 @@ class TestLokiQueryApi(APIBaseTest):"limit": ["100"],}assert kwargs["headers"]["X-Team-ID"] == str(self.team.pk)- assert kwargs["auth"] == ("reader", "secret")+ assert kwargs["auth"] == HTTPBasicAuth("reader", "secret")assert kwargs["timeout"] == 12assert kwargs["data"] is Nonediff --git a/products/metrics/backend/alerts/evaluation.py b/products/metrics/backend/alerts/evaluation.pyindex 392a37c6a4bd..a7f0b76f3d52 100644--- a/products/metrics/backend/alerts/evaluation.py+++ b/products/metrics/backend/alerts/evaluation.py@@ -31,6 +31,7 @@ from django.conf import settingsimport structlogfrom pydantic import ValidationError+from posthog.clickhouse.client.connection import ClickHouseUserfrom posthog.dataclasses import frozenfrom posthog.models import Teamfrom posthog.temporal.alerts.admission import hold_evaluation_slot, release_evaluation_slot@@ -265,6 +266,7 @@ def _query(team: Team, expr: str, *, at: datetime, timeout_seconds: float) -> li"/api/v1/query",data={"query": expr, "time": str(at.timestamp()), "timeout": f"{int(timeout_seconds)}s"},timeout=timeout_seconds,+ ch_user=ClickHouseUser.ALERTS_PLATFORM_METRICS,)data = payload.get("data") or {}diff --git a/products/metrics/backend/alerts/freshness.py b/products/metrics/backend/alerts/freshness.pyindex 5a55b1ea76d9..dc293ee16931 100644--- a/products/metrics/backend/alerts/freshness.py+++ b/products/metrics/backend/alerts/freshness.py@@ -77,8 +77,8 @@ def _bounded_logs_client(send_receive_timeout: int) -> AbstractContextManager:Mirrors the file-backed branch of `get_pool`, so a token-authenticated user stays token-aware.`make_ch_pool` is cached on its arguments, so every call reuses one pool."""- kwargs = get_kwargs_for_client(workload=Workload.LOGS, ch_user=ClickHouseUser.DEFAULT)- creds = connection_creds(Workload.LOGS, ClickHouseUser.DEFAULT)+ kwargs = get_kwargs_for_client(workload=Workload.LOGS, ch_user=ClickHouseUser.ALERTS_PLATFORM_METRICS)+ creds = connection_creds(Workload.LOGS, ClickHouseUser.ALERTS_PLATFORM_METRICS)if creds is not None and is_file_backed_user(creds, kwargs.get("user")):kwargs.pop("password", None)pool = make_ch_pool(diff --git a/products/metrics/backend/promql.py b/products/metrics/backend/promql.pyindex fbd571110add..59116e4b6f05 100644--- a/products/metrics/backend/promql.py+++ b/products/metrics/backend/promql.py@@ -12,6 +12,7 @@ from typing import Anyimport requestsfrom posthog.api.snuffle_proxy import SnuffleNotConfiguredError, snuffle_request+from posthog.clickhouse.client.connection import ClickHouseUserfrom posthog.models import Teamfrom products.metrics.backend.facade.contracts import MetricPoint, MetricSeries, PromQLQueryError@@ -78,12 +79,13 @@ def snuffle_payload(data: dict[str, str] | None = None,params: dict[str, list[str]] | None = None,timeout: float | None = None,+ ch_user: ClickHouseUser | None = None,) -> dict[str, Any]:"""Send one Prometheus API request to Snuffle and return its JSON body. Errors are safe to show."""if path in _QUERY_PATHS and data and data.get("query"):record_promql_query(team, data["query"])try:- response = snuffle_request(team.pk, method, path, data=data, params=params, timeout=timeout)+ response = snuffle_request(team.pk, method, path, data=data, params=params, timeout=timeout, ch_user=ch_user)except SnuffleNotConfiguredError as exc:# Deployment config, not the query, so an alert waiting on it is not counted as broken.raise PromQLQueryError("PromQL queries are not available on this PostHog instance.", transient=True) from excdiff --git a/products/metrics/backend/tests/test_alerts.py b/products/metrics/backend/tests/test_alerts.pyindex 2fe4b835c4e8..9b40fadecff1 100644--- a/products/metrics/backend/tests/test_alerts.py+++ b/products/metrics/backend/tests/test_alerts.py@@ -15,6 +15,7 @@ from parameterized import parameterizedfrom pydantic import ValidationErrorfrom rest_framework import status+from posthog.clickhouse.client.connection import ClickHouseUserfrom posthog.models import Teamfrom posthog.models.integration import Integrationfrom posthog.models.scoping import team_scope@@ -220,6 +221,7 @@ class TestMetricsAlertEvaluation(APIBaseTest):assert self._kinds(first) == {"a": AlertEventKind.FIRING, "b": AlertEventKind.FIRING}assert self.snuffle.call_args.kwargs["data"]["time"] == str((self.due_at - timedelta(seconds=30)).timestamp())+ assert self.snuffle.call_args.kwargs["ch_user"] == ClickHouseUser.ALERTS_PLATFORM_METRICSself._next_check(first)self.snuffle.return_value = _response(diff --git a/products/metrics/backend/tests/test_prometheus_api.py b/products/metrics/backend/tests/test_prometheus_api.pyindex 1a7423e70da4..51195fff278a 100644--- a/products/metrics/backend/tests/test_prometheus_api.py+++ b/products/metrics/backend/tests/test_prometheus_api.py@@ -1,5 +1,6 @@from typing import Any+import pytestfrom posthog.test.base import APIBaseTestfrom unittest.mock import MagicMock, patch@@ -7,8 +8,12 @@ from django.test import override_settingsimport requestsfrom parameterized import parameterized+from requests.auth import HTTPBasicAuthfrom rest_framework import status+from posthog.api.snuffle_proxy import snuffle_request+from posthog.clickhouse.client import connection+from posthog.clickhouse.client.connection import ClickHouseCredentials, ClickHouseUserfrom posthog.models.personal_api_key import PersonalAPIKeyfrom posthog.models.utils import generate_random_token_personal, hash_key_value@@ -23,6 +28,44 @@ def _upstream() -> MagicMock:return response+ "registered,ch_user,expected",+ [+ (+ True,+ ClickHouseUser.ALERTS_PLATFORM_METRICS,+ [("alerts_platform_metrics", "token-1"), ("alerts_platform_metrics", "token-2")],+ ),+ (False, ClickHouseUser.ALERTS_PLATFORM_METRICS, [("reader", "secret"), ("reader", "secret")]),+ (True, None, [("reader", "secret"), ("reader", "secret")]),+ ],+)+def test_snuffle_request_authenticates_as_the_registered_user_with_its_current_token(+ settings, monkeypatch, tmp_path, registered, ch_user, expected+):+ settings.SNUFFLE_APM_URL = "http://snuffle.test:9091"+ settings.SNUFFLE_APM_USER = "reader"+ settings.SNUFFLE_APM_PASSWORD = "secret"+ token = tmp_path / "token"+ users = {ClickHouseUser.DEFAULT: ClickHouseCredentials(user="default", password="x")}+ if registered:+ users[ClickHouseUser.ALERTS_PLATFORM_METRICS] = ClickHouseCredentials(+ user="alerts_platform_metrics", password="", password_file=str(token)+ )+ monkeypatch.setattr(connection, "__user_dict", users)+ request = MagicMock(return_value=_upstream())+ monkeypatch.setattr("posthog.api.snuffle_proxy.internal_requests.request", request)++ sent = []+ for rotation in ("token-1", "token-2"):+ token.write_text(rotation)+ snuffle_request(1, "POST", "/api/v1/query", data={"query": "up"}, ch_user=ch_user)+ auth = request.call_args.kwargs["auth"]+ sent.append((auth.username, auth.password))++ assert sent == expected++@override_settings(SNUFFLE_APM_URL="http://snuffle.test:9091", SNUFFLE_APM_USER="reader", SNUFFLE_APM_PASSWORD="secret")class TestPrometheusQueryApi(APIBaseTest):def setUp(self):@@ -43,7 +86,7 @@ class TestPrometheusQueryApi(APIBaseTest):assert url == "http://snuffle.test:9091/api/v1/query"assert kwargs["params"] == {"query": ["sum(rate(http_requests_total[5m]))"]}assert kwargs["headers"]["X-Team-ID"] == str(self.team.pk)- assert kwargs["auth"] == ("reader", "secret")+ assert kwargs["auth"] == HTTPBasicAuth("reader", "secret")@parameterized.expand([