:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(metrics): write shared histogram quantiles as one histogram_quantiles call (#115606)
Frank Hamand192f9ecaa60a
diff --git a/products/metrics/backend/promql.py b/products/metrics/backend/promql.pyindex cee6105d300b..5791a5f8c5cc 100644--- a/products/metrics/backend/promql.py+++ b/products/metrics/backend/promql.py@@ -23,6 +23,12 @@ CLAUSE_LABEL = "clause"# builderToPromql sets the label with `label_replace(<series>, "clause", "<alias>", "", "")`. Without# that call, a `clause` label belongs to the user's data._SETS_CLAUSE_LABEL = re.compile(r'label_replace\s*\(.*,\s*"clause"\s*,\s*"', re.DOTALL)+# builderToPromql writes quantiles of the same buckets as `histogram_quantiles("clause", 0.5, 0.9, <buckets>)`,+# so Snuffle reads the buckets one time. The label holds the quantile. The quantiles belong to the+# clauses a, b, c, ... in order.+_SHARED_HISTOGRAM_QUANTILES = re.compile(+ r'^\s*histogram_quantiles\s*\(\s*"clause"\s*((?:,\s*[0-9.eE+-]+\s*)+),', re.DOTALL+)class PromQLQueryError(ValueError):@@ -45,6 +51,27 @@ def _error_message(response: requests.Response) -> str:return str(payload.get("error") or f"The PromQL query failed with status {response.status_code}.")+def _quantile_clauses(expr: str) -> dict[float, str]:+ """The clause alias of each quantile in a `histogram_quantiles("clause", ...)` query."""+ match = _SHARED_HISTOGRAM_QUANTILES.match(expr)+ if not match:+ return {}+ try:+ quantiles = [float(value) for value in match.group(1).split(",")[1:]]+ except ValueError:+ return {}+ return {quantile: chr(ord("a") + index) for index, quantile in enumerate(quantiles)}+++def _clause_alias(value: str | None, quantile_clauses: dict[float, str]) -> str | None:+ if value is None or not quantile_clauses:+ return value+ try:+ return quantile_clauses.get(float(value), value)+ except ValueError:+ return value++def snuffle_payload(team: Team,method: str,@@ -104,12 +131,13 @@ def run_promql_range(raise PromQLQueryError(f"A PromQL range query cannot return a {result_type} result.")tzinfo = team.timezone_info- marks_clauses = bool(_SETS_CLAUSE_LABEL.search(expr))+ quantile_clauses = _quantile_clauses(expr)+ marks_clauses = bool(quantile_clauses) or bool(_SETS_CLAUSE_LABEL.search(expr))rows: list[tuple[dict[str, str], str | None, str | None, list[MetricPoint]]] = []for result in results:labels = {str(key): str(value) for key, value in (result.get("metric") or {}).items()}metric_name = labels.pop("__name__", None)- clause = labels.pop(CLAUSE_LABEL, None) if marks_clauses else None+ clause = _clause_alias(labels.pop(CLAUSE_LABEL, None), quantile_clauses) if marks_clauses else Nonepoints = [MetricPoint(time=dt.datetime.fromtimestamp(float(timestamp), tz=tzinfo).isoformat(),diff --git a/products/metrics/backend/tests/test_metrics_query_languages.py b/products/metrics/backend/tests/test_metrics_query_languages.pyindex 7eec63e62b6c..75c8c52da193 100644--- a/products/metrics/backend/tests/test_metrics_query_languages.py+++ b/products/metrics/backend/tests/test_metrics_query_languages.py@@ -531,6 +531,27 @@ class TestMetricsPromQLMode(APIBaseTest):with pytest.raises(ExposedHogQLError):runner.calculate()+ def test_shared_histogram_quantiles_map_each_quantile_to_its_clause(self) -> None:+ start = int(dt.datetime(2026, 9, 19, 11, 0, tzinfo=dt.UTC).timestamp())+ payload = {+ "status": "success",+ "data": {+ "resultType": "matrix",+ "result": [+ {"metric": {"job": "api", "clause": quantile}, "values": [[start, value]]}+ for quantile, value in (("0.9", "9"), ("0.5", "5"))+ ],+ },+ }+ promql = 'histogram_quantiles("clause", 0.9, 0.5, sum by (le, job) (rate(x_bucket)))'+ with patch("posthog.api.snuffle_proxy.internal_requests.request", return_value=_snuffle_response(200, payload)):+ results = self._runner(promql).calculate().results++ assert {series.clause: (series.labels, series.points[0].value) for series in results} == {+ "a": ({"job": "api"}, 9.0),+ "b": ({"job": "api"}, 5.0),+ }+def test_empty_scalar_result_has_no_series(self) -> None:payload = {"status": "success", "data": {"resultType": "scalar", "result": []}}with patch("posthog.api.snuffle_proxy.internal_requests.request", return_value=_snuffle_response(200, payload)):diff --git a/products/metrics/frontend/queryLanguages/__fixtures__/builder_sql.json b/products/metrics/frontend/queryLanguages/__fixtures__/builder_sql.jsonindex 0a76d2302060..875facfca056 100644--- a/products/metrics/frontend/queryLanguages/__fixtures__/builder_sql.json+++ b/products/metrics/frontend/queryLanguages/__fixtures__/builder_sql.json@@ -159,6 +159,25 @@},"sql": "SELECT\n time,\n service_name,\n if(\n idx > length(bounds),\n bounds[length(bounds)],\n if(idx > 1, bounds[idx - 1], least(0.0, bounds[1]))\n + (bounds[idx] - if(idx > 1, bounds[idx - 1], least(0.0, bounds[1])))\n * (rank - (cumulative[idx] - counts[idx])) / counts[idx]\n ) AS value\nFROM (\n SELECT\n time,\n service_name,\n bounds,\n counts,\n arrayCumSum(counts) AS cumulative,\n 0.5 * arraySum(counts) AS rank,\n arrayFirstIndex(c -> c >= rank, cumulative) AS idx\n FROM (\n SELECT\n toStartOfInterval(s.sample_timestamp, {interval}) AS time,\n ser.group_0 AS service_name,\n any(s.histogram_bounds) AS bounds,\n uniqExactIf(s.histogram_bounds, notEmpty(s.histogram_bounds)) AS layouts,\n arrayMap(x -> ifNull(x, 0.0), sumForEach(s.contribution_counts)) AS counts\n FROM (\n SELECT\n timestamp AS sample_timestamp,\n series_fingerprint,\n histogram_bounds,\n multiIf(\n aggregation_temporality = 'delta', counts_f,\n empty(prev_counts), arrayMap(x -> 0.0, counts_f),\n length(prev_counts) != length(counts_f), counts_f,\n arrayAll((c, p) -> c >= p, counts_f, prev_counts), arrayMap((c, p) -> c - p, counts_f, prev_counts),\n counts_f\n ) AS contribution_counts\n FROM (\n SELECT\n timestamp,\n series_fingerprint,\n aggregation_temporality,\n histogram_bounds,\n arrayMap(x -> toFloat(x), histogram_counts) AS counts_f,\n lagInFrame(arrayMap(x -> toFloat(x), histogram_counts)) OVER (\n PARTITION BY series_fingerprint\n ORDER BY timestamp ASC\n ROWS BETWEEN 1 PRECEDING AND 1 PRECEDING\n ) AS prev_counts\n FROM posthog.metrics\n WHERE metric_name = 'http.server.duration'\n AND timestamp >= {date_from} - toIntervalSecond(greatest({interval_seconds}, 300))\n AND timestamp < {date_to}\n AND notEmpty(histogram_counts)\n )\n ) AS s\n LEFT JOIN (\n SELECT\n series_fingerprint,\n any(toString(service_name)) AS group_0\n FROM posthog.metric_series\n WHERE metric_name = 'http.server.duration'\n AND last_seen >= {date_from} - toIntervalHour(1)\n GROUP BY series_fingerprint\n ) AS ser ON s.series_fingerprint = ser.series_fingerprint\n WHERE s.sample_timestamp >= {date_from}\n GROUP BY time, service_name\n )\n WHERE arraySum(counts) > 0\n AND throwIf(layouts > 1, 'The series have different histogram bounds. Add filters so that all series use the same buckets.') = 0\n)\nORDER BY time"},+ "histogram p50 and p99 of the same buckets": {+ "builder": {+ "clauses": [+ {+ "name": "a",+ "aggregation": "histogram_quantile",+ "metricName": "http.server.duration",+ "quantile": 0.5+ },+ {+ "name": "b",+ "aggregation": "histogram_quantile",+ "metricName": "http.server.duration",+ "quantile": 0.99+ }+ ]+ },+ "sql": "SELECT\n time,\n clause,\n if(\n idx > length(bounds),\n bounds[length(bounds)],\n if(idx > 1, bounds[idx - 1], least(0.0, bounds[1]))\n + (bounds[idx] - if(idx > 1, bounds[idx - 1], least(0.0, bounds[1])))\n * (rank - (cumulative[idx] - counts[idx])) / counts[idx]\n ) AS value\nFROM (\n SELECT\n time,\n clause,\n bounds,\n counts,\n arrayCumSum(counts) AS cumulative,\n 0.5 * arraySum(counts) AS rank,\n arrayFirstIndex(c -> c >= rank, cumulative) AS idx\n FROM (\n SELECT\n toStartOfInterval(s.sample_timestamp, {interval}) AS time,\n 'a' AS clause,\n any(s.histogram_bounds) AS bounds,\n uniqExactIf(s.histogram_bounds, notEmpty(s.histogram_bounds)) AS layouts,\n arrayMap(x -> ifNull(x, 0.0), sumForEach(s.contribution_counts)) AS counts\n FROM (\n SELECT\n timestamp AS sample_timestamp,\n series_fingerprint,\n histogram_bounds,\n multiIf(\n aggregation_temporality = 'delta', counts_f,\n empty(prev_counts), arrayMap(x -> 0.0, counts_f),\n length(prev_counts) != length(counts_f), counts_f,\n arrayAll((c, p) -> c >= p, counts_f, prev_counts), arrayMap((c, p) -> c - p, counts_f, prev_counts),\n counts_f\n ) AS contribution_counts\n FROM (\n SELECT\n timestamp,\n series_fingerprint,\n aggregation_temporality,\n histogram_bounds,\n arrayMap(x -> toFloat(x), histogram_counts) AS counts_f,\n lagInFrame(arrayMap(x -> toFloat(x), histogram_counts)) OVER (\n PARTITION BY series_fingerprint\n ORDER BY timestamp ASC\n ROWS BETWEEN 1 PRECEDING AND 1 PRECEDING\n ) AS prev_counts\n FROM posthog.metrics\n WHERE metric_name = 'http.server.duration'\n AND timestamp >= {date_from} - toIntervalSecond(greatest({interval_seconds}, 300))\n AND timestamp < {date_to}\n AND notEmpty(histogram_counts)\n )\n ) AS s\n WHERE s.sample_timestamp >= {date_from}\n GROUP BY time\n )\n WHERE arraySum(counts) > 0\n AND throwIf(layouts > 1, 'The series have different histogram bounds. Add filters so that all series use the same buckets.') = 0\n)\nUNION ALL\nSELECT\n time,\n clause,\n if(\n idx > length(bounds),\n bounds[length(bounds)],\n if(idx > 1, bounds[idx - 1], least(0.0, bounds[1]))\n + (bounds[idx] - if(idx > 1, bounds[idx - 1], least(0.0, bounds[1])))\n * (rank - (cumulative[idx] - counts[idx])) / counts[idx]\n ) AS value\nFROM (\n SELECT\n time,\n clause,\n bounds,\n counts,\n arrayCumSum(counts) AS cumulative,\n 0.99 * arraySum(counts) AS rank,\n arrayFirstIndex(c -> c >= rank, cumulative) AS idx\n FROM (\n SELECT\n toStartOfInterval(s.sample_timestamp, {interval}) AS time,\n 'b' AS clause,\n any(s.histogram_bounds) AS bounds,\n uniqExactIf(s.histogram_bounds, notEmpty(s.histogram_bounds)) AS layouts,\n arrayMap(x -> ifNull(x, 0.0), sumForEach(s.contribution_counts)) AS counts\n FROM (\n SELECT\n timestamp AS sample_timestamp,\n series_fingerprint,\n histogram_bounds,\n multiIf(\n aggregation_temporality = 'delta', counts_f,\n empty(prev_counts), arrayMap(x -> 0.0, counts_f),\n length(prev_counts) != length(counts_f), counts_f,\n arrayAll((c, p) -> c >= p, counts_f, prev_counts), arrayMap((c, p) -> c - p, counts_f, prev_counts),\n counts_f\n ) AS contribution_counts\n FROM (\n SELECT\n timestamp,\n series_fingerprint,\n aggregation_temporality,\n histogram_bounds,\n arrayMap(x -> toFloat(x), histogram_counts) AS counts_f,\n lagInFrame(arrayMap(x -> toFloat(x), histogram_counts)) OVER (\n PARTITION BY series_fingerprint\n ORDER BY timestamp ASC\n ROWS BETWEEN 1 PRECEDING AND 1 PRECEDING\n ) AS prev_counts\n FROM posthog.metrics\n WHERE metric_name = 'http.server.duration'\n AND timestamp >= {date_from} - toIntervalSecond(greatest({interval_seconds}, 300))\n AND timestamp < {date_to}\n AND notEmpty(histogram_counts)\n )\n ) AS s\n WHERE s.sample_timestamp >= {date_from}\n GROUP BY time\n )\n WHERE arraySum(counts) > 0\n AND throwIf(layouts > 1, 'The series have different histogram bounds. Add filters so that all series use the same buckets.') = 0\n)"+ },"eq filter": {"builder": {"clauses": [diff --git a/products/metrics/frontend/queryLanguages/builderToPromql.ts b/products/metrics/frontend/queryLanguages/builderToPromql.tsindex a353822187c9..531a57c27cb1 100644--- a/products/metrics/frontend/queryLanguages/builderToPromql.ts+++ b/products/metrics/frontend/queryLanguages/builderToPromql.ts@@ -6,6 +6,7 @@ import {type BuilderQuery,CLAUSE_LABEL,type ConversionResult,+ clauseAlias,normalizeClause,normalizeLabelKey,} from './types'@@ -166,6 +167,39 @@ function substituteFormula(}}+/**+ * `histogram_quantiles("clause", 0.5, 0.9, buckets)` for series that are quantiles of the same buckets.+ * Snuffle reads the buckets one time for this form; a union of `histogram_quantile` calls reads them+ * once for each series, on a much slower path. The label holds the quantile, and the backend maps it+ * back to the alias, so this form is only used when the aliases are the defaults a, b, c, ...+ */+function sharedHistogramQuantiles(clauses: BuilderClause[], exprs: Map<string, ClauseExpr>): PromExpr | null {+ const quantiles: PromExpr[] = []+ let buckets: PromExpr | null = null+ for (const [index, clause] of clauses.entries()) {+ const expr = exprs.get(clause.name)!.expr+ if (clause.name !== clauseAlias(index) || expr.type !== 'call' || expr.func !== 'histogram_quantile') {+ return null+ }+ const [quantile, clauseBuckets] = expr.args+ if (quantile.type !== 'number' || quantiles.some((q) => q.type === 'number' && q.value === quantile.value)) {+ return null+ }+ if (buckets && printPromQL(buckets) !== printPromQL(clauseBuckets)) {+ return null+ }+ buckets = clauseBuckets+ quantiles.push(quantile)+ }+ return (+ buckets && {+ type: 'call',+ func: 'histogram_quantiles',+ args: [{ type: 'string', value: CLAUSE_LABEL }, ...quantiles, buckets],+ }+ )+}+export function builderToPromql(query: BuilderQuery): ConversionResult<string> {const issues: string[] = []const clauses = query.clauses.filter((clause) => clause.metricName.trim())@@ -202,6 +236,11 @@ export function builderToPromql(query: BuilderQuery): ConversionResult<string> {return { value: printPromQL(exprs.get(clauses[0].name)!.expr), issues }}+ const shared = sharedHistogramQuantiles(clauses, exprs)+ if (shared) {+ return { value: printPromQL(shared), issues }+ }+// Several series without a formula: label each one with its alias, so the union keeps them apart.const labelled: PromExpr[] = clauses.map((clause) => ({type: 'call',diff --git a/products/metrics/frontend/queryLanguages/promqlToBuilder.ts b/products/metrics/frontend/queryLanguages/promqlToBuilder.tsindex a8a6c08cea9b..b121c5802910 100644--- a/products/metrics/frontend/queryLanguages/promqlToBuilder.ts+++ b/products/metrics/frontend/queryLanguages/promqlToBuilder.ts@@ -384,6 +384,20 @@ const labelledClause = (expr: PromExpr): { alias: string; expr: PromExpr } | nulreturn null}+/** `histogram_quantiles("clause", 0.5, 0.9, buckets)`, the form builderToPromql gives quantiles of the same buckets. */+const sharedHistogramQuantiles = (expr: PromExpr): PromExpr[] | null => {+ expr = unwrapParens(expr)+ if (expr.type !== 'call' || expr.func !== 'histogram_quantiles' || expr.args.length < 3) {+ return null+ }+ const [label, ...rest] = expr.args.map(unwrapParens)+ const buckets = rest.pop()!+ if (label.type !== 'string' || label.value !== CLAUSE_LABEL || rest.some((arg) => arg.type !== 'number')) {+ return null+ }+ return rest.map((quantile) => ({ type: 'call', func: 'histogram_quantile', args: [quantile, buckets] }))+}+const orOperands = (expr: PromExpr): PromExpr[] => {const unwrapped = unwrapParens(expr)return unwrapped.type === 'binary' && unwrapped.op === 'or' && !unwrapped.matching@@ -405,6 +419,12 @@ export function promqlToBuilder(text: string): ConversionResult<BuilderQuery> {const issues: string[] = []const builder = new FormulaBuilder(issues)+ const quantiles = sharedHistogramQuantiles(parsed)+ if (quantiles) {+ quantiles.forEach((quantile, index) => builder.addClause(quantile, clauseAlias(index)))+ return builder.clauses.length ? { value: { clauses: builder.clauses }, issues } : { value: null, issues }+ }+const operands = orOperands(parsed)if (operands.length > 1) {const labelled = operands.map(labelledClause)diff --git a/products/metrics/frontend/queryLanguages/queryLanguages.test.ts b/products/metrics/frontend/queryLanguages/queryLanguages.test.tsindex a998cc64a189..ba9104d7141f 100644--- a/products/metrics/frontend/queryLanguages/queryLanguages.test.ts+++ b/products/metrics/frontend/queryLanguages/queryLanguages.test.ts@@ -61,6 +61,17 @@ export const LOSSLESS_BUILDER_FIXTURES: Record<string, BuilderQuery> = {}),],},+ 'histogram p50 and p99 of the same buckets': {+ clauses: [+ clause({ metricName: 'http.server.duration', aggregation: 'histogram_quantile', quantile: 0.5 }),+ clause({+ name: 'b',+ metricName: 'http.server.duration',+ aggregation: 'histogram_quantile',+ quantile: 0.99,+ }),+ ],+ },'eq filter': {clauses: [clause({ metricName: 'queue_depth', filters: [{ key: 'service_name', op: 'eq', value: 'api' }] })],},@@ -793,6 +804,10 @@ describe('metrics query languages', () => {'sum(rate({"http.server.request.count", "http.response.status_code"=~"5.*"})) / sum(rate({"http.server.request.count"})) * 100',],['parenthesized formula', '(sum(queue_depth) + sum(retry_queue_depth)) / 2'],+ [+ 'histogram p50 and p99 of the same buckets',+ 'histogram_quantiles("clause", 0.5, 0.99, sum by (le) (rate({"http.server.duration_bucket"})))',+ ],['two series without formula','label_replace(sum(queue_depth), "clause", "a", "", "") or label_replace(avg({"process.cpu.utilization"}), "clause", "b", "", "")',@@ -801,6 +816,32 @@ describe('metrics query languages', () => {expect(builderToPromql(LOSSLESS_BUILDER_FIXTURES[name])).toEqual({ value: expected, issues: [] })})+ it.each<[string, BuilderClause[]]>([+ [+ 'different buckets',+ [+ clause({ metricName: 'x', aggregation: 'histogram_quantile', quantile: 0.5 }),+ clause({ name: 'b', metricName: 'y', aggregation: 'histogram_quantile', quantile: 0.9 }),+ ],+ ],+ [+ 'aliases that are not the defaults',+ [+ clause({ metricName: 'x', aggregation: 'histogram_quantile', quantile: 0.5 }),+ clause({ name: 'c', metricName: 'x', aggregation: 'histogram_quantile', quantile: 0.9 }),+ ],+ ],+ [+ 'the same quantile twice',+ [+ clause({ metricName: 'x', aggregation: 'histogram_quantile', quantile: 0.5 }),+ clause({ name: 'b', metricName: 'x', aggregation: 'histogram_quantile', quantile: 0.5 }),+ ],+ ],+ ])('keeps the label_replace union for histogram quantiles with %s', (_name, clauses) => {+ expect(builderToPromql({ clauses }).value).toMatch(/^label_replace\(histogram_quantile\(/)+ })+it('spreads an ungrouped series over a grouped one with on() group_left', () => {const result = builderToPromql({clauses: [