:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
import datetime as dt
from dataclasses import replace
from functools import cached_property
from math import ceil
from time import monotonic
from typing import TYPE_CHECKING
import posthoganalytics
from posthog.schema import CachedLogsQueryResponse, LogsQuery
from posthog.hogql import ast
from posthog.hogql.constants import HogQLGlobalSettings
from posthog.hogql.parser import parse_expr, parse_select
from posthog.hogql.query import execute_hogql_query
from posthog.clickhouse.client.connection import Workload
from posthog.clickhouse.client.limit import app_org_concurrency_slot
from posthog.dataclasses import frozen
from posthog.exceptions import ClickHouseQueryTimeOut
from posthog.hogql_queries.query_runner import AnalyticsQueryRunner
from posthog.utils import get_instance_region
from products.logs.backend.log_patterns import (
ERROR_SEVERITIES,
LogSample,
MinedPattern,
_env,
encoded_pattern_member_size,
group_stored_patterns,
mine_patterns,
)
from products.logs.backend.logs_query_runner import LogsQueryResponse, LogsQueryRunnerMixin
if TYPE_CHECKING:
from posthog.models import User
from posthog.query_cache import QueryCache
_TimeSlice = tuple[dt.datetime, dt.datetime]
STORED_PATTERN_HEAD_MAX_BYTES = 65536
@frozen
class _VersionCoverage:
total: int
covered: int
version: int
class PatternsQueryRunner(AnalyticsQueryRunner[LogsQueryResponse], LogsQueryRunnerMixin):
"""Mines log templates (Drain3) from a bounded, deterministic sample of the matching logs.
The sampling has to respect two constraints at once:
* A uniform per-row sampling predicate cannot be short-circuited by LIMIT — ClickHouse
has to scan (and decompress `body` for) the whole eligible range to find the matches.
On high-volume windows that blows the wall-clock budget, and a timed-out scan that
returns partially is worse than an error: it silently mines patterns from whatever
handful of rows the scan reached. So the rows eligible for sampling are capped at
`LOGS_PATTERNS_MAX_SCAN_ROWS` via stratified time slices — evenly spaced sub-windows
(`LOGS_PATTERNS_SLICE_COUNT` of them, the last aligned to the window end) that keep
the sample spread across the window while bounding what the query may read. As a
backstop, `timeout_overflow_mode="throw"` turns any remaining timeout into an error
instead of a truncated sample.
* Results should be reproducible: the same filters over the same window must mine the
same patterns. The sampling predicate hashes the row's immutable `uuid`
(`cityHash64(uuid) % divisor = 0`) rather than using `rand()`, so the sample — and
therefore the mined patterns — is a pure function of the data and the divisor.
`sampled` is True whenever fewer rows were scanned than the window held — i.e. whenever
the reported per-pattern counts are extrapolated estimates rather than exact tallies,
whether the scan was narrowed by hash-mod sampling or by the time slices (or both).
"""
query: LogsQuery
cached_response: CachedLogsQueryResponse
use_stored_patterns: bool = True
_query_deadline: float | None = None
request_deadline: float | None = None
_request_deadline_binds: bool = False
def get_cache_payload(self) -> dict:
payload = super().get_cache_payload()
# Set after construction, so the base payload cannot see it. Keyed only when off, so
# stored-pattern runs keep their cache entries.
if not self.use_stored_patterns:
payload["use_stored_patterns"] = False
return payload
@cached_property
def settings(self) -> HogQLGlobalSettings:
# Bytes are intentionally uncapped: a hard `max_bytes_to_read` + "throw" cap 500s on
# wide windows (the symbol-stats read-cap cliff). The scan is bounded by the time
# slices instead; timeout_overflow_mode="throw" guarantees a slow scan surfaces as
# an error rather than silently mining from a partial sample.
return HogQLGlobalSettings(
max_execution_time=_env("LOGS_PATTERNS_MAX_EXECUTION_TIME", 60, int),
max_bytes_to_read=None,
read_overflow_mode=None,
timeout_overflow_mode="throw",
)
@cached_property
def _sample_limit(self) -> int:
return _env("LOGS_PATTERNS_SAMPLE_LIMIT", 10000, int)
@cached_property
def _scan_budget(self) -> int:
return _env("LOGS_PATTERNS_MAX_SCAN_ROWS", 2_000_000, int)
@cached_property
def _slice_count(self) -> int:
return _env("LOGS_PATTERNS_SLICE_COUNT", 12, int)
@cached_property
def _sparkline_bucket_count(self) -> int:
return _env("LOGS_PATTERNS_SPARKLINE_BUCKETS", 24, int)
def validate_query_runner_access(self, user: "User") -> bool:
# Defensive: this runner is invoked directly via the logs API, never through the generic
# /api/projects/:id/query/ endpoint. Mirror LogsQueryRunner and refuse user-initiated
# generic-query access so it can't silently bypass that gate if ever registered.
from products.access_control.backend.facade.user_access_control import UserAccessControlError
raise UserAccessControlError("logs", "viewer")
def _calculate(self) -> LogsQueryResponse:
reason = "comparison" if not self.use_stored_patterns else "flag_disabled"
if self.use_stored_patterns and self._stored_patterns_enabled:
self._start_deadline()
coverage = self._version_coverage
total, covered, version = coverage.total, coverage.covered, coverage.version
if total and covered * 100 >= total * 99:
return self._calculate_stored_patterns(total, version, covered)
reason = "insufficient_version_coverage" if total else "empty_window"
else:
cached = self.__dict__.get("_version_coverage")
total = cached.total if cached is not None else self._count()
return self._calculate_body_patterns(total, reason)
def _start_deadline(self) -> None:
if self._query_deadline is None:
self._query_deadline = monotonic() + max(1, self.settings.max_execution_time or 60)
@cached_property
def _version_coverage(self) -> "_VersionCoverage":
versions = self._execute(
parse_select(
"SELECT pattern_version, count(), countIf(trim(pattern) != '') FROM logs WHERE {where} GROUP BY pattern_version",
placeholders={"where": self._where_with_timestamp()},
)
).results
candidates = [(int(row[2]), int(row[0])) for row in versions if int(row[0]) > 0]
covered, version = max(candidates, default=(0, 0))
return _VersionCoverage(total=sum(int(row[1]) for row in versions), covered=covered, version=version)
def stored_version(self) -> int | None:
if not (self.use_stored_patterns and self._stored_patterns_enabled):
return None
self._start_deadline()
with app_org_concurrency_slot(self.team, task_id=self.query_id):
coverage = self._version_coverage
return coverage.version if coverage.total and coverage.covered * 100 >= coverage.total * 99 else None
@cached_property
def _stored_patterns_enabled(self) -> bool:
# only_evaluate_locally keeps this check off the network: it gates an interactive query, so
# a flags-service round trip (plus its retry) would delay every patterns request. An
# inconclusive local evaluation therefore means "off", which is body mining as before.
# `group_properties` carries the organization id so an org-level rollout still resolves locally.
team_id = str(self.team.pk)
person_properties = {"team_id": team_id, "region": get_instance_region() or "DEV"}
if self.user:
person_properties["email"] = self.user.email
return bool(
posthoganalytics.feature_enabled(
"logs_patterns_query_v2",
str(self.user.distinct_id) if self.user else team_id,
person_properties=person_properties,
groups={"organization": str(self.team.organization_id)},
group_properties={"organization": {"id": str(self.team.organization_id)}},
only_evaluate_locally=True,
send_feature_flag_events=False,
)
)
def _calculate_body_patterns(self, total: int, reason: str) -> LogsQueryResponse:
slices = _time_slices(
self.query_date_range.date_from(),
self.query_date_range.date_to(),
total=total,
scan_budget=self._scan_budget,
slice_count=self._slice_count,
)
# The slice row count is exact (counts don't read `body`, so a second count is cheap),
# which keeps the divisor honest even when log volume is bursty across the window.
pool = self._count(slices) if slices is not None else total
divisor = _sample_divisor(pool, self._sample_limit)
response = self._execute(self._sample_query(divisor, slices))
samples = [
LogSample(
body=row[0],
severity_text=row[1],
service_name=row[2],
timestamp=row[3].replace(tzinfo=dt.UTC),
)
for row in response.results
]
# Sparkline buckets: when the scan is slice-bounded the slices ARE the buckets — they're
# evenly spaced by construction and rows between them were never eligible, so uniform
# buckets would render misleading zeros in the gaps. Unsliced windows bucket uniformly.
buckets = (
slices
if slices is not None
else _uniform_buckets(
self.query_date_range.date_from(), self.query_date_range.date_to(), self._sparkline_bucket_count
)
)
patterns = mine_patterns(samples, buckets=buckets)
# `sampled` mirrors the exact condition `_serialize` uses to extrapolate: it's True
# whenever fewer rows were scanned than the window held (hash-mod sampling OR time-slice
# bounding), so the flag can never diverge from whether the reported counts are estimates.
scanned = len(samples)
return LogsQueryResponse(
results={
"patterns": [_serialize(p, total_count=total, scanned_count=scanned) for p in patterns],
"scanned_count": scanned,
"total_count": total,
"sampled": scanned < total,
"sample_coverage_pct": round(pool / total * 100, 2) if total else 100.0,
"sparkline_buckets": [{"start": start.isoformat(), "end": end.isoformat()} for start, end in buckets],
"source": "body_mining",
"pattern_version": None,
"fallback_reason": reason,
"pattern_coverage_pct": None,
"represented_count": None,
"remainder_count": None,
}
)
def _stored_where(self, version: int) -> ast.Expr:
return parse_expr(
"{where} AND pattern_version = {version} AND trim(pattern) != ''",
placeholders={"where": self._where_with_timestamp(), "version": ast.Constant(value=version)},
)
def _calculate_stored_patterns(self, total: int, version: int, covered: int) -> LogsQueryResponse:
head_limit = min(10000, max(1, _env("LOGS_PATTERNS_HEAD_LIMIT", 10000, int)))
head = self._execute(
parse_select(
"SELECT topK({head_limit})(pattern) FROM logs WHERE {where}",
placeholders={"head_limit": ast.Constant(value=head_limit), "where": self._stored_where(version)},
)
).results[0][0]
bounded_head = []
head_bytes = 3
for member in head:
member_bytes = encoded_pattern_member_size(member)
if head_bytes + member_bytes <= STORED_PATTERN_HEAD_MAX_BYTES:
bounded_head.append(member)
head_bytes += member_bytes
buckets = _uniform_buckets(
self.query_date_range.date_from(), self.query_date_range.date_to(), self._sparkline_bucket_count
)
placeholders: dict[str, ast.Expr] = {
"where": self._stored_where(version),
"head": ast.Constant(value=bounded_head),
"services_limit": ast.Constant(value=_env("LOGS_PATTERNS_MAX_SERVICES", 4, int)),
"head_limit": ast.Constant(value=head_limit),
}
bucket_sql = []
for index, (start, end) in enumerate(buckets):
placeholders[f"start_{index}"] = ast.Constant(value=start)
placeholders[f"end_{index}"] = ast.Constant(value=end)
bucket_sql.append(f"countIf(timestamp >= {{start_{index}}} AND timestamp < {{end_{index}}})")
rows = self._execute(
parse_select(
"SELECT pattern, count(), min(timestamp), max(timestamp), "
"sumMap([lower(severity_text)], [1]), "
"groupUniqArrayArray({services_limit})([service_name]), "
f"[{', '.join(bucket_sql)}] "
"FROM logs WHERE {where} AND pattern IN {head} GROUP BY pattern "
"ORDER BY count() DESC, pattern LIMIT {head_limit}",
placeholders=placeholders,
)
).results
patterns = group_stored_patterns([_stamped_pattern(row, version) for row in rows], total_count=total)
patterns = self._stored_examples(patterns, version)
represented = sum(pattern.count for pattern in patterns)
return LogsQueryResponse(
results={
"patterns": [_serialize(pattern, total_count=total, scanned_count=total) for pattern in patterns],
"total_count": total,
"scanned_count": total,
"sampled": False,
"sample_coverage_pct": 100.0,
"sparkline_buckets": [{"start": start.isoformat(), "end": end.isoformat()} for start, end in buckets],
"source": "stored_patterns",
"pattern_version": version,
"fallback_reason": None,
"pattern_coverage_pct": round(covered / total * 100, 2),
"represented_count": represented,
"remainder_count": max(0, total - represented),
}
)
def _stored_examples(self, patterns: list[MinedPattern], version: int) -> list[MinedPattern]:
if not patterns:
return []
max_examples = _env("LOGS_PATTERNS_MAX_EXAMPLES", 10, int)
if max_examples <= 0:
return patterns
member_groups = {
member: group_id for group_id, pattern in enumerate(patterns, start=1) for member in pattern.match_patterns
}
# `LIMIT n BY` caps rows, not distinct bodies, and the table's sort order puts the rows of a
# repeated line next to each other, so one body can fill a group's whole quota. Read a
# bounded multiple of the quota and keep the distinct bodies here. Grouping on `body` in the
# query would decompress it for every matching row instead of stopping at the rows read
# first, which is the cost this query is shaped to avoid.
fetch_per_group = max_examples * 4
rows = self._execute(
parse_select(
"SELECT transform(pattern, {patterns}, {group_ids}, 0) AS pattern_group, "
"substringUTF8(body, 1, 4096), severity_text, service_name, timestamp "
"FROM logs WHERE {where} AND pattern IN {patterns} "
"LIMIT {fetch_per_group} BY pattern_group LIMIT {limit}",
placeholders={
"where": self._stored_where(version),
"patterns": ast.Constant(value=list(member_groups)),
"group_ids": ast.Constant(value=list(member_groups.values())),
"fetch_per_group": ast.Constant(value=fetch_per_group),
"limit": ast.Constant(value=len(patterns) * fetch_per_group),
},
)
).results
examples: dict[int, list[LogSample]] = {}
for row in rows:
group = examples.setdefault(row[0], [])
if len(group) >= max_examples or any(example.body == row[1] for example in group):
continue
group.append(
LogSample(
body=row[1], severity_text=row[2], service_name=row[3], timestamp=row[4].replace(tzinfo=dt.UTC)
)
)
return [
replace(pattern, examples=examples.get(group_id, [])) for group_id, pattern in enumerate(patterns, start=1)
]
def run(self, *args, **kwargs) -> LogsQueryResponse | CachedLogsQueryResponse:
response = super().run(*args, **kwargs)
assert isinstance(response, LogsQueryResponse | CachedLogsQueryResponse)
return response
def to_query(self) -> ast.SelectQuery:
# Canonical (unsampled) form for the base runner's query contract; _calculate picks
# the runtime divisor and slices.
return self._sample_query(divisor=1, slices=None)
def _execute(self, query: ast.SelectQuery | ast.SelectSetQuery):
settings = self.settings.model_copy()
deadline = min((d for d in (self._query_deadline, self.request_deadline) if d is not None), default=None)
self._request_deadline_binds = False
if deadline is not None:
remaining = int(deadline - monotonic())
own_limit = settings.max_execution_time or remaining
self._request_deadline_binds = deadline == self.request_deadline and remaining <= own_limit
if remaining < 1:
raise ClickHouseQueryTimeOut()
# The deadline bounds the whole run. It must not give one query more time than its own limit.
settings.max_execution_time = min(remaining, own_limit)
response = execute_hogql_query(
query_type="LogsQuery",
query=query,
modifiers=self.modifiers,
team=self.team,
workload=Workload.LOGS,
timings=self.timings,
limit_context=self.limit_context,
settings=settings,
)
if deadline is not None and monotonic() >= deadline:
raise ClickHouseQueryTimeOut()
return response
def _record_breaker_failure(self, cache_manager: "QueryCache", exc: Exception) -> None:
# A failure under the request deadline depends on how long the earlier queries in the request took.
# It does not repeat on a fresh request, so it must not open the breaker for this query.
if self._request_deadline_binds:
return
super()._record_breaker_failure(cache_manager, exc)
def _count(self, slices: list[_TimeSlice] | None = None) -> int:
response = self._execute(
parse_select(
"SELECT count() FROM logs WHERE {where}",
placeholders={"where": self._where_with_timestamp(slices)},
)
)
return int(response.results[0][0]) if response.results else 0
def _where_with_timestamp(self, slices: list[_TimeSlice] | None = None) -> ast.Expr:
# LogsFilterBuilder.where() filters at day-precision via time_bucket; add explicit
# per-row timestamp bounds (half-open) so the sample matches the requested window.
exprs: list[ast.Expr] = [
self.where(),
parse_expr(
"timestamp >= {date_from} AND timestamp < {date_to}",
placeholders={
"date_from": ast.Constant(value=self.query_date_range.date_from()),
"date_to": ast.Constant(value=self.query_date_range.date_to()),
},
),
]
if slices is not None:
# Slice bounds are on `timestamp` (in the sort key), so ClickHouse prunes the
# granules outside the slices instead of scanning the whole window.
exprs.append(
ast.Or(
exprs=[
parse_expr(
"timestamp >= {slice_from} AND timestamp < {slice_to}",
placeholders={
"slice_from": ast.Constant(value=slice_from),
"slice_to": ast.Constant(value=slice_to),
},
)
for slice_from, slice_to in slices
]
)
)
return ast.And(exprs=exprs)
def _sample_query(self, divisor: int, slices: list[_TimeSlice] | None = None) -> ast.SelectQuery:
# Deterministic even-random sampling: cityHash64(uuid) is uniform and fixed per row,
# so `% divisor = 0` keeps ~1/divisor of rows spread evenly across the slices AND the
# same rows on every run. divisor == 1 means no sampling, so we skip the predicate.
where: ast.Expr = self._where_with_timestamp(slices)
if divisor > 1:
where = ast.And(
exprs=[
where,
parse_expr(
"cityHash64(uuid) % {divisor} = 0",
placeholders={"divisor": ast.Constant(value=divisor)},
),
]
)
query = parse_select(
"""
SELECT body, severity_text, service_name, timestamp
FROM logs
WHERE {where}
LIMIT {limit}
""",
placeholders={"where": where, "limit": ast.Constant(value=self._sample_limit)},
)
assert isinstance(query, ast.SelectQuery)
return query
def _time_slices(
date_from: dt.datetime,
date_to: dt.datetime,
*,
total: int,
scan_budget: int,
slice_count: int,
) -> list[_TimeSlice] | None:
"""Evenly spaced sub-windows covering ~scan_budget/total of [date_from, date_to).
Returns None when the whole window fits the budget. The last slice is aligned to end at
date_to so the freshest logs are always eligible for the sample.
"""
if total <= scan_budget:
return None
window = date_to - date_from
if window <= dt.timedelta(0):
return None
slice_count = max(1, slice_count)
step = window / slice_count
width = window * (scan_budget / total) / slice_count
slices = []
for i in range(slice_count):
end = date_from + step * (i + 1)
slices.append((end - width, end))
return slices
def _uniform_buckets(date_from: dt.datetime, date_to: dt.datetime, count: int) -> list[_TimeSlice]:
count = max(1, count)
step = (date_to - date_from) / count
return [(date_from + step * i, date_from + step * (i + 1)) for i in range(count)]
def _sample_divisor(total: int, sample_limit: int) -> int:
# Round up so total / divisor <= sample_limit: the hash-modulo predicate alone bounds the
# sample and LIMIT never truncates the subset in (biased) read order.
if total <= sample_limit:
return 1
return ceil(total / sample_limit)
def _serialize(pattern: MinedPattern, *, total_count: int, scanned_count: int) -> dict:
# Extrapolate sample counts to the full window so consumers don't have to; when the
# window wasn't sampled the estimates are the exact counts.
def estimate(sample_count: int) -> int:
if scanned_count >= total_count:
return sample_count
return round(sample_count / scanned_count * total_count)
return {
"pattern": pattern.pattern,
"count": pattern.count,
"estimated_count": estimate(pattern.count),
"volume_share_pct": pattern.volume_share_pct,
"error_count": pattern.error_count,
"estimated_error_count": estimate(pattern.error_count),
"first_seen": pattern.first_seen.isoformat(),
"last_seen": pattern.last_seen.isoformat(),
"examples": [
{
"body": example.body,
"severity_text": example.severity_text,
"service_name": example.service_name,
"timestamp": example.timestamp.isoformat(),
}
for example in pattern.examples
],
"services": pattern.services,
"sparkline": [estimate(c) for c in pattern.bucket_counts],
# Raw sample counts: severity dominance is a proportion, which is scale-invariant,
# so extrapolating the map would add payload without changing what it says.
"severity_counts": pattern.severity_counts,
"match_regex": pattern.match_regex,
"match_literal": pattern.match_literal,
"match_patterns": pattern.match_patterns,
"pattern_version": pattern.pattern_version,
}
def _stamped_pattern(row: list, version: int) -> MinedPattern:
severities = dict(zip(row[4][0], row[4][1]))
return MinedPattern(
pattern=row[0],
count=int(row[1]),
volume_share_pct=0,
error_count=sum(count for severity, count in severities.items() if severity in ERROR_SEVERITIES),
first_seen=row[2].replace(tzinfo=dt.UTC),
last_seen=row[3].replace(tzinfo=dt.UTC),
examples=[],
services=sorted(service for service in row[5] if service),
bucket_counts=row[6],
severity_counts=severities,
match_regex=None,
match_literal=None,
match_patterns=[row[0]],
pattern_version=version,
)