Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(alerts-platform): run metrics alert queries as their own clickhouse user (#130477)

Jimmy Zhu6b9b56f50fc6
diff --git a/posthog/api/snuffle_proxy.py b/posthog/api/snuffle_proxy.py
index 471dea4a75af..52d3d1f31e32 100644
--- a/posthog/api/snuffle_proxy.py
+++ b/posthog/api/snuffle_proxy.py
@@ -20,6 +20,7 @@ from django.http import HttpResponse, QueryDict
import requests
import structlog
from drf_spectacular.utils import extend_schema
+from requests.auth import HTTPBasicAuth
from rest_framework import status, viewsets
from rest_framework.decorators import action
from rest_framework.parsers import FormParser
@@ -27,6 +28,7 @@ from rest_framework.request import Request
from posthog.api.routing import TeamAndOrgViewSetMixin
from posthog.api_queries_budget import debit
+from posthog.clickhouse.client.connection import ClickHouseUser, Workload, connection_creds
from posthog.hogql_queries.query_runner import api_queries_budget_enforcement_enabled, get_api_queries_budget_status
from posthog.permissions import PostHogFeatureFlagPermission
from posthog.rate_limit import ClickHouseBurstRateThrottle, ClickHouseSustainedRateThrottle
@@ -118,6 +120,15 @@ class SnuffleNotConfiguredError(Exception):
"""Raised when no Snuffle URL is configured for this deployment."""
+def _snuffle_auth(ch_user: ClickHouseUser | None) -> HTTPBasicAuth | None:
+ # A token-backed user's file rotates, so the credential is read for each request.
+ if ch_user is not None and (creds := connection_creds(Workload.LOGS, ch_user)) is not None:
+ return HTTPBasicAuth(creds.user, creds.read_password())
+ return (
+ HTTPBasicAuth(settings.SNUFFLE_APM_USER, settings.SNUFFLE_APM_PASSWORD) if settings.SNUFFLE_APM_USER else None
+ )
+
+
def snuffle_request(
team_id: int,
method: str,
@@ -127,17 +138,19 @@ def snuffle_request(
data: dict[str, list[str]] | dict[str, str] | None = None,
accept: str = "application/json",
timeout: float | None = None,
+ ch_user: ClickHouseUser | None = None,
) -> requests.Response:
"""Send one request to Snuffle as `team_id` and debit the bytes it read from the query budget.
- `upstream_path` is the Snuffle path, for example ``/api/v1/query_range``. Network errors from
+ `upstream_path` is the Snuffle path, for example ``/api/v1/query_range``. Snuffle connects to
+ ClickHouse as `ch_user` when it is a registered logs cluster user. Network errors from
`requests` propagate, so each caller can map them to its own error format.
"""
base_url = settings.SNUFFLE_APM_URL
if not base_url:
raise SnuffleNotConfiguredError("PromQL/LogQL query API is not configured")
headers = {TEAM_ID_HEADER: str(team_id), "Accept": accept}
- auth = (settings.SNUFFLE_APM_USER, settings.SNUFFLE_APM_PASSWORD) if settings.SNUFFLE_APM_USER else None
+ auth = _snuffle_auth(ch_user)
upstream = internal_requests.request(
method,
f"{base_url.rstrip('/')}{upstream_path}",
diff --git a/posthog/clickhouse/client/connection.py b/posthog/clickhouse/client/connection.py
index fd641c59afc7..69f813b72686 100644
--- a/posthog/clickhouse/client/connection.py
+++ b/posthog/clickhouse/client/connection.py
@@ -160,6 +160,8 @@ class ClickHouseUser(StrEnum):
ALERTS_PLATFORM_INSIGHT = "alerts_platform_insight"
# Exists on the logs cluster only. Pass it with Workload.LOGS, never as a query tag.
ALERTS_PLATFORM_LOGS = "alerts_platform_logs"
+ # Exists on the logs cluster only. Snuffle connects as it for the platform's PromQL queries.
+ ALERTS_PLATFORM_METRICS = "alerts_platform_metrics"
# The platform's own bookkeeping, shared by every source: its check history and the default
# user of its evaluation worker. A source's checks keep their own user, so a new source never
# takes from another source's budget.
@@ -451,7 +453,9 @@ def get_http_client(**overrides):
# Named users that exist on the logs cluster. Every other named user exists on the main cluster
# only, and sync_execute assigns some of them (APP, API) from tags without checking the workload.
-LOGS_CLUSTER_USERS: frozenset[ClickHouseUser] = frozenset({ClickHouseUser.ALERTS_PLATFORM_LOGS})
+LOGS_CLUSTER_USERS: frozenset[ClickHouseUser] = frozenset(
+ {ClickHouseUser.ALERTS_PLATFORM_LOGS, ClickHouseUser.ALERTS_PLATFORM_METRICS}
+)
def connection_creds(workload: Workload, ch_user: ClickHouseUser) -> ClickHouseCredentials | None:
diff --git a/products/logs/backend/test/test_loki_api.py b/products/logs/backend/test/test_loki_api.py
index 1368c2b1c818..2e1ef1274d36 100644
--- a/products/logs/backend/test/test_loki_api.py
+++ b/products/logs/backend/test/test_loki_api.py
@@ -5,6 +5,7 @@ from django.test import SimpleTestCase, override_settings
import requests
from parameterized import parameterized
+from requests.auth import HTTPBasicAuth
from rest_framework import status
import posthog.api.snuffle_proxy as snuffle_proxy
@@ -67,7 +68,7 @@ class TestLokiQueryApi(APIBaseTest):
"limit": ["100"],
}
assert kwargs["headers"]["X-Team-ID"] == str(self.team.pk)
- assert kwargs["auth"] == ("reader", "secret")
+ assert kwargs["auth"] == HTTPBasicAuth("reader", "secret")
assert kwargs["timeout"] == 12
assert kwargs["data"] is None
diff --git a/products/metrics/backend/alerts/evaluation.py b/products/metrics/backend/alerts/evaluation.py
index 392a37c6a4bd..a7f0b76f3d52 100644
--- a/products/metrics/backend/alerts/evaluation.py
+++ b/products/metrics/backend/alerts/evaluation.py
@@ -31,6 +31,7 @@ from django.conf import settings
import structlog
from pydantic import ValidationError
+from posthog.clickhouse.client.connection import ClickHouseUser
from posthog.dataclasses import frozen
from posthog.models import Team
from posthog.temporal.alerts.admission import hold_evaluation_slot, release_evaluation_slot
@@ -265,6 +266,7 @@ def _query(team: Team, expr: str, *, at: datetime, timeout_seconds: float) -> li
"/api/v1/query",
data={"query": expr, "time": str(at.timestamp()), "timeout": f"{int(timeout_seconds)}s"},
timeout=timeout_seconds,
+ ch_user=ClickHouseUser.ALERTS_PLATFORM_METRICS,
)
data = payload.get("data") or {}
diff --git a/products/metrics/backend/alerts/freshness.py b/products/metrics/backend/alerts/freshness.py
index 5a55b1ea76d9..dc293ee16931 100644
--- a/products/metrics/backend/alerts/freshness.py
+++ b/products/metrics/backend/alerts/freshness.py
@@ -77,8 +77,8 @@ def _bounded_logs_client(send_receive_timeout: int) -> AbstractContextManager:
Mirrors the file-backed branch of `get_pool`, so a token-authenticated user stays token-aware.
`make_ch_pool` is cached on its arguments, so every call reuses one pool.
"""
- kwargs = get_kwargs_for_client(workload=Workload.LOGS, ch_user=ClickHouseUser.DEFAULT)
- creds = connection_creds(Workload.LOGS, ClickHouseUser.DEFAULT)
+ kwargs = get_kwargs_for_client(workload=Workload.LOGS, ch_user=ClickHouseUser.ALERTS_PLATFORM_METRICS)
+ creds = connection_creds(Workload.LOGS, ClickHouseUser.ALERTS_PLATFORM_METRICS)
if creds is not None and is_file_backed_user(creds, kwargs.get("user")):
kwargs.pop("password", None)
pool = make_ch_pool(
diff --git a/products/metrics/backend/promql.py b/products/metrics/backend/promql.py
index fbd571110add..59116e4b6f05 100644
--- a/products/metrics/backend/promql.py
+++ b/products/metrics/backend/promql.py
@@ -12,6 +12,7 @@ from typing import Any
import requests
from posthog.api.snuffle_proxy import SnuffleNotConfiguredError, snuffle_request
+from posthog.clickhouse.client.connection import ClickHouseUser
from posthog.models import Team
from products.metrics.backend.facade.contracts import MetricPoint, MetricSeries, PromQLQueryError
@@ -78,12 +79,13 @@ def snuffle_payload(
data: dict[str, str] | None = None,
params: dict[str, list[str]] | None = None,
timeout: float | None = None,
+ ch_user: ClickHouseUser | None = None,
) -> dict[str, Any]:
"""Send one Prometheus API request to Snuffle and return its JSON body. Errors are safe to show."""
if path in _QUERY_PATHS and data and data.get("query"):
record_promql_query(team, data["query"])
try:
- response = snuffle_request(team.pk, method, path, data=data, params=params, timeout=timeout)
+ response = snuffle_request(team.pk, method, path, data=data, params=params, timeout=timeout, ch_user=ch_user)
except SnuffleNotConfiguredError as exc:
# Deployment config, not the query, so an alert waiting on it is not counted as broken.
raise PromQLQueryError("PromQL queries are not available on this PostHog instance.", transient=True) from exc
diff --git a/products/metrics/backend/tests/test_alerts.py b/products/metrics/backend/tests/test_alerts.py
index 2fe4b835c4e8..9b40fadecff1 100644
--- a/products/metrics/backend/tests/test_alerts.py
+++ b/products/metrics/backend/tests/test_alerts.py
@@ -15,6 +15,7 @@ from parameterized import parameterized
from pydantic import ValidationError
from rest_framework import status
+from posthog.clickhouse.client.connection import ClickHouseUser
from posthog.models import Team
from posthog.models.integration import Integration
from posthog.models.scoping import team_scope
@@ -220,6 +221,7 @@ class TestMetricsAlertEvaluation(APIBaseTest):
assert self._kinds(first) == {"a": AlertEventKind.FIRING, "b": AlertEventKind.FIRING}
assert self.snuffle.call_args.kwargs["data"]["time"] == str((self.due_at - timedelta(seconds=30)).timestamp())
+ assert self.snuffle.call_args.kwargs["ch_user"] == ClickHouseUser.ALERTS_PLATFORM_METRICS
self._next_check(first)
self.snuffle.return_value = _response(
diff --git a/products/metrics/backend/tests/test_prometheus_api.py b/products/metrics/backend/tests/test_prometheus_api.py
index 1a7423e70da4..51195fff278a 100644
--- a/products/metrics/backend/tests/test_prometheus_api.py
+++ b/products/metrics/backend/tests/test_prometheus_api.py
@@ -1,5 +1,6 @@
from typing import Any
+import pytest
from posthog.test.base import APIBaseTest
from unittest.mock import MagicMock, patch
@@ -7,8 +8,12 @@ from django.test import override_settings
import requests
from parameterized import parameterized
+from requests.auth import HTTPBasicAuth
from rest_framework import status
+from posthog.api.snuffle_proxy import snuffle_request
+from posthog.clickhouse.client import connection
+from posthog.clickhouse.client.connection import ClickHouseCredentials, ClickHouseUser
from posthog.models.personal_api_key import PersonalAPIKey
from posthog.models.utils import generate_random_token_personal, hash_key_value
@@ -23,6 +28,44 @@ def _upstream() -> MagicMock:
return response
+ "registered,ch_user,expected",
+ [
+ (
+ True,
+ ClickHouseUser.ALERTS_PLATFORM_METRICS,
+ [("alerts_platform_metrics", "token-1"), ("alerts_platform_metrics", "token-2")],
+ ),
+ (False, ClickHouseUser.ALERTS_PLATFORM_METRICS, [("reader", "secret"), ("reader", "secret")]),
+ (True, None, [("reader", "secret"), ("reader", "secret")]),
+ ],
+)
+def test_snuffle_request_authenticates_as_the_registered_user_with_its_current_token(
+ settings, monkeypatch, tmp_path, registered, ch_user, expected
+):
+ settings.SNUFFLE_APM_URL = "http://snuffle.test:9091"
+ settings.SNUFFLE_APM_USER = "reader"
+ settings.SNUFFLE_APM_PASSWORD = "secret"
+ token = tmp_path / "token"
+ users = {ClickHouseUser.DEFAULT: ClickHouseCredentials(user="default", password="x")}
+ if registered:
+ users[ClickHouseUser.ALERTS_PLATFORM_METRICS] = ClickHouseCredentials(
+ user="alerts_platform_metrics", password="", password_file=str(token)
+ )
+ monkeypatch.setattr(connection, "__user_dict", users)
+ request = MagicMock(return_value=_upstream())
+ monkeypatch.setattr("posthog.api.snuffle_proxy.internal_requests.request", request)
+
+ sent = []
+ for rotation in ("token-1", "token-2"):
+ token.write_text(rotation)
+ snuffle_request(1, "POST", "/api/v1/query", data={"query": "up"}, ch_user=ch_user)
+ auth = request.call_args.kwargs["auth"]
+ sent.append((auth.username, auth.password))
+
+ assert sent == expected
+
+
@override_settings(SNUFFLE_APM_URL="http://snuffle.test:9091", SNUFFLE_APM_USER="reader", SNUFFLE_APM_PASSWORD="secret")
class TestPrometheusQueryApi(APIBaseTest):
def setUp(self):
@@ -43,7 +86,7 @@ class TestPrometheusQueryApi(APIBaseTest):
assert url == "http://snuffle.test:9091/api/v1/query"
assert kwargs["params"] == {"query": ["sum(rate(http_requests_total[5m]))"]}
assert kwargs["headers"]["X-Team-ID"] == str(self.team.pk)
- assert kwargs["auth"] == ("reader", "secret")
+ assert kwargs["auth"] == HTTPBasicAuth("reader", "secret")
@parameterized.expand(
[