Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(logs): report distinct log patterns per team in the usage report (#118598)

Jon McCallum1f94565edce1
diff --git a/posthog/tasks/test/__snapshots__/test_usage_report.ambr b/posthog/tasks/test/__snapshots__/test_usage_report.ambr
index 084dca5c8bfa..42a99145d7f3 100644
--- a/posthog/tasks/test/__snapshots__/test_usage_report.ambr
+++ b/posthog/tasks/test/__snapshots__/test_usage_report.ambr
@@ -38,6 +38,23 @@
GROUP BY team_id
'''
# ---
+# name: TestFeatureFlagsUsageReport.test_usage_report_decide_requests.100
+ '''
+
+ SELECT team_id,
+ argMax(patterns, pattern_version) AS count
+ FROM
+ (SELECT team_id,
+ pattern_version,
+ uniq(pattern) AS patterns
+ FROM logs_pattern_buckets_distributed
+ WHERE time_bucket >= '2022-01-10 00:00:00'
+ AND time_bucket < '2022-01-10 23:59:59'
+ GROUP BY team_id,
+ pattern_version)
+ GROUP BY team_id
+ '''
+# ---
# name: TestFeatureFlagsUsageReport.test_usage_report_decide_requests.11
'''
diff --git a/posthog/tasks/test/test_usage_report.py b/posthog/tasks/test/test_usage_report.py
index 72b4ce804013..df93f659b59b 100644
--- a/posthog/tasks/test/test_usage_report.py
+++ b/posthog/tasks/test/test_usage_report.py
@@ -40,6 +40,7 @@ from posthog.hogql.query import execute_hogql_query
from posthog.clickhouse.client import sync_execute
from posthog.clickhouse.client.connection import ClickHouseUser
from posthog.clickhouse.logs.logs32 import TABLE_NAME as LOGS_LOCAL_TABLE
+from posthog.clickhouse.logs.logs_pattern_buckets import TABLE_NAME as LOGS_PATTERN_BUCKETS_TABLE
from posthog.clickhouse.query_tagging import tag_queries
from posthog.cloud_utils import TEST_clear_instance_license_cache
from posthog.hogql_queries.events_query_runner import EventsQueryRunner
@@ -72,6 +73,7 @@ from posthog.tasks.usage_report import (
get_teams_with_ai_credits_used_in_period,
get_teams_with_billable_event_count_in_period,
get_teams_with_event_count_with_groups_in_period,
+ get_teams_with_logs_distinct_patterns_in_period,
get_teams_with_posthog_code_credits_used_in_period,
get_teams_with_query_metric,
get_teams_with_sdk_logs_records_in_period,
@@ -3747,6 +3749,47 @@ class TestHogFunctionUsageReports(ClickhouseDestroyTablesMixin, TestCase, Clickh
assert get_teams_with_sdk_logs_records_in_period(begin, end, [team_id]) == expected
assert get_teams_with_sdk_logs_records_in_period(begin, end, []) == {sdk: [] for sdk in expected}
+ def test_logs_distinct_patterns_count_latest_version_across_period(self) -> None:
+ self._setup_teams()
+ sync_execute(f"TRUNCATE TABLE IF EXISTS {LOGS_PATTERN_BUCKETS_TABLE}")
+ begin = datetime(2026, 10, 16, tzinfo=UTC)
+ end = begin + timedelta(days=1)
+ team_id = self.org_1_team_1.id
+ other_team_id = self.org_1_team_2.id
+
+ def row(
+ team: int, bucket: datetime, version: int, pattern: str, service: str = "api", severity: str = "info"
+ ) -> str:
+ return json.dumps(
+ {
+ "team_id": team,
+ "time_bucket": bucket.strftime("%Y-%m-%d %H:%M:%S"),
+ "service_name": service,
+ "namespace": "",
+ "environment": "",
+ "severity_text": severity,
+ "pattern_version": version,
+ "pattern": pattern,
+ "retention_days": 30,
+ "log_count": 1,
+ }
+ )
+
+ rows = [
+ row(team_id, begin, 5, "old a"),
+ row(team_id, begin, 5, "old b"),
+ row(team_id, begin, 5, "old c"),
+ row(team_id, begin + timedelta(hours=12), 6, "a"),
+ row(team_id, begin + timedelta(hours=13), 6, "a", service="worker", severity="error"),
+ row(team_id, end - timedelta(minutes=5), 6, "b"),
+ row(team_id, begin - timedelta(minutes=5), 6, "before"),
+ row(team_id, end, 6, "after"),
+ row(other_team_id, begin, 5, "x"),
+ ]
+ sync_execute(f"INSERT INTO {LOGS_PATTERN_BUCKETS_TABLE} FORMAT JSONEachRow\n" + "\n".join(rows))
+
+ assert dict(get_teams_with_logs_distinct_patterns_in_period(begin, end)) == {team_id: 2, other_team_id: 1}
+
@parameterized.expand(
[
# Per-signal MB is floored to whole decimal MB, and the billable combined metric is
diff --git a/posthog/tasks/usage_report.py b/posthog/tasks/usage_report.py
index 25181a7b8fb1..5cf7aa0ca5fb 100644
--- a/posthog/tasks/usage_report.py
+++ b/posthog/tasks/usage_report.py
@@ -396,6 +396,9 @@ class UsageReportCounters:
android_logs_records_in_period: int
flutter_logs_records_in_period: int
ruby_logs_records_in_period: int
+ # Distinct log patterns, a measure of how varied a team's logs are. Report-only: it tells real
+ # usage apart from a test ping and does not feed any billed field.
+ logs_distinct_patterns_in_period: int
# Distributed Tracing (APM). apm_tracing_mb_in_period is report-only, like logs_mb_in_period.
apm_tracing_bytes_in_period: int
@@ -2640,6 +2643,41 @@ def get_teams_with_logs_records_in_period(
)
+@timed_log()
+@retry(tries=QUERY_RETRIES, delay=QUERY_RETRY_DELAY, backoff=QUERY_RETRY_BACKOFF)
+def get_teams_with_logs_distinct_patterns_in_period(
+ begin: datetime,
+ end: datetime,
+) -> TeamUsageRows:
+ """
+ Returns the number of distinct log patterns each team sent in the period.
+
+ Reads the `logs_pattern_buckets` rollup, never raw logs. A pattern only means the same shape
+ within one masker version, so each team counts the patterns of the latest `pattern_version` it
+ sent in the period. A period that spans a masker change counts only the newer version's
+ patterns, which undercounts that one day instead of counting each shape twice.
+
+ `uniq` keeps memory bounded for a team that emits a very large number of shapes. It is exact
+ for small counts, which is where a test ping and real usage differ.
+ """
+ with tags_context(product=Product.LOGS, feature=Feature.USAGE_REPORT):
+ return sync_execute(
+ """
+ SELECT team_id, argMax(patterns, pattern_version) AS count
+ FROM (
+ SELECT team_id, pattern_version, uniq(pattern) AS patterns
+ FROM logs_pattern_buckets_distributed
+ WHERE time_bucket >= %(begin)s AND time_bucket < %(end)s
+ GROUP BY team_id, pattern_version
+ )
+ GROUP BY team_id
+ """,
+ {"begin": begin, "end": end},
+ workload=Workload.LOGS,
+ settings=CH_BILLING_SETTINGS,
+ )
+
+
# Maps the `telemetry.sdk.name` resource attribute (the SDK identifier, set on every log record) to
# the report field suffix used in `UsageReportCounters` / `_get_all_usage_data` keys. posthog-js sets
# `web`; posthog-rails sets `posthog-ruby`; the mobile SDKs set their package name.
@@ -3228,6 +3266,9 @@ def _get_all_usage_data(
"teams_with_android_logs_records_in_period": sdk_logs_by_suffix["android"],
"teams_with_flutter_logs_records_in_period": sdk_logs_by_suffix["flutter"],
"teams_with_ruby_logs_records_in_period": sdk_logs_by_suffix["ruby"],
+ "teams_with_logs_distinct_patterns_in_period": get_teams_with_logs_distinct_patterns_in_period(
+ period_start, period_end
+ ),
"teams_with_apm_tracing_bytes_in_period": apm_tracing_usage["bytes"],
"teams_with_apm_tracing_spans_in_period": apm_tracing_usage["spans"],
"teams_with_metrics_bytes_in_period": metrics_usage["bytes"],
@@ -3517,6 +3558,7 @@ def _get_team_report(all_data: dict[str, Any], team: Team) -> UsageReportCounter
android_logs_records_in_period=all_data["teams_with_android_logs_records_in_period"].get(team.id, 0),
flutter_logs_records_in_period=all_data["teams_with_flutter_logs_records_in_period"].get(team.id, 0),
ruby_logs_records_in_period=all_data["teams_with_ruby_logs_records_in_period"].get(team.id, 0),
+ logs_distinct_patterns_in_period=all_data["teams_with_logs_distinct_patterns_in_period"].get(team.id, 0),
apm_tracing_bytes_in_period=apm_tracing_bytes_in_period,
apm_tracing_spans_in_period=all_data["teams_with_apm_tracing_spans_in_period"].get(team.id, 0),
apm_tracing_mb_in_period=int(apm_tracing_bytes_in_period // 1_000_000),
diff --git a/posthog/temporal/tests/usage_report/test_aggregator.py b/posthog/temporal/tests/usage_report/test_aggregator.py
index 8f330ace0815..36af147f021b 100644
--- a/posthog/temporal/tests/usage_report/test_aggregator.py
+++ b/posthog/temporal/tests/usage_report/test_aggregator.py
@@ -26,6 +26,7 @@ from posthog.tasks.usage_report import (
)
from posthog.temporal.tests.usage_report.test_aggregate_activity import _instance_metadata
from posthog.temporal.usage_report.aggregator import (
+ add_pre_logs_distinct_patterns_patch_defaults,
add_pre_sandbox_compute_patch_defaults,
build_manifest,
iter_chunk_lines,
@@ -166,6 +167,13 @@ def test_load_all_data_defaults_compute_for_pre_patch_workflow_history() -> None
assert all_data["teams_with_sandbox_compute_memory_mib_seconds_in_period"] == {}
+def test_load_all_data_defaults_logs_distinct_patterns_for_pre_patch_workflow_history() -> None:
+ all_data: dict[str, dict[int, int]] = {}
+ add_pre_logs_distinct_patterns_patch_defaults(all_data, [])
+
+ assert all_data == {"teams_with_logs_distinct_patterns_in_period": {}}
+
+
def test_load_all_data_does_not_default_compute_for_patched_workflow_history() -> None:
all_data: dict[str, dict[int, int]] = {}
results = [RunQueryToS3Result(query_name="sandbox_compute_usage", s3_key="unused", duration_ms=1)]
diff --git a/posthog/temporal/tests/usage_report/test_workflow.py b/posthog/temporal/tests/usage_report/test_workflow.py
index 5c2dc8cbf6c3..4ab001d07b58 100644
--- a/posthog/temporal/tests/usage_report/test_workflow.py
+++ b/posthog/temporal/tests/usage_report/test_workflow.py
@@ -46,11 +46,13 @@ from posthog.temporal.usage_report.types import (
)
from posthog.temporal.usage_report.workflow import (
AGGREGATION_QUEUE_PATCH_ID,
+ LOGS_DISTINCT_PATTERNS_QUERY_NAME,
PARALLEL_COUNTER_QUERIES_PATCH_ID,
QUERY_CONCURRENCY,
SANDBOX_COMPUTE_QUERY_NAME,
USAGE_COUNTER_SERVICE_PATCH_ID,
RunUsageReportsWorkflow,
+ _queries_for_logs_distinct_patterns_patch,
_queries_for_sandbox_compute_patch,
build_context,
)
@@ -68,6 +70,15 @@ def test_sandbox_compute_query_is_versioned_for_existing_histories() -> None:
assert SANDBOX_COMPUTE_QUERY_NAME in [spec.name for spec in _queries_for_sandbox_compute_patch(True)]
+def test_logs_distinct_patterns_query_is_versioned_for_existing_histories() -> None:
+ assert LOGS_DISTINCT_PATTERNS_QUERY_NAME not in [
+ spec.name for spec in _queries_for_logs_distinct_patterns_patch(QUERIES, False)
+ ]
+ assert LOGS_DISTINCT_PATTERNS_QUERY_NAME in [
+ spec.name for spec in _queries_for_logs_distinct_patterns_patch(QUERIES, True)
+ ]
+
+
@parameterized.expand(
[
# (day_offset, now, expected_date, expected_completeness) — intraday run mid-day reports today
diff --git a/posthog/temporal/usage_report/activities.py b/posthog/temporal/usage_report/activities.py
index 8ec83bae329c..ee71b62b0cab 100644
--- a/posthog/temporal/usage_report/activities.py
+++ b/posthog/temporal/usage_report/activities.py
@@ -26,6 +26,7 @@ from posthog.tasks.usage_report import get_instance_metadata, iter_reports_with_
from posthog.temporal.common.heartbeat import Heartbeater
from posthog.temporal.common.metrics import ExecutionTimeRecorder
from posthog.temporal.usage_report.aggregator import (
+ add_pre_logs_distinct_patterns_patch_defaults,
add_pre_sandbox_compute_patch_defaults,
get_org_user_counts,
iter_org_reports,
@@ -247,6 +248,7 @@ async def aggregate_and_chunk_org_reports(inputs: AggregateInputs) -> AggregateR
check_cancelled()
all_data = load_all_data(inputs.query_results)
add_pre_sandbox_compute_patch_defaults(all_data, inputs.query_results)
+ add_pre_logs_distinct_patterns_patch_defaults(all_data, inputs.query_results)
check_cancelled()
counter_report = load_usage_counter_report(inputs)
all_data.update({field: dict(rows) for field, rows in counter_report.counts.items()})
diff --git a/posthog/temporal/usage_report/aggregator.py b/posthog/temporal/usage_report/aggregator.py
index 18c4c7628b53..276701079f8b 100644
--- a/posthog/temporal/usage_report/aggregator.py
+++ b/posthog/temporal/usage_report/aggregator.py
@@ -55,6 +55,7 @@ _SANDBOX_COMPUTE_DESTINATION_KEYS = (
"teams_with_sandbox_compute_cpu_millicore_seconds_in_period",
"teams_with_sandbox_compute_memory_mib_seconds_in_period",
)
+_LOGS_DISTINCT_PATTERNS_QUERY_NAME = "teams_with_logs_distinct_patterns_in_period"
def load_usage_counter_report(inputs: AggregateInputs) -> UsageCounterReport:
@@ -94,6 +95,13 @@ def add_pre_sandbox_compute_patch_defaults(
all_data[key] = {}
+def add_pre_logs_distinct_patterns_patch_defaults(
+ all_data: dict[str, dict[int, int]], query_results: list[RunQueryToS3Result]
+) -> None:
+ if not any(result.query_name == _LOGS_DISTINCT_PATTERNS_QUERY_NAME for result in query_results):
+ all_data[_LOGS_DISTINCT_PATTERNS_QUERY_NAME] = {}
+
+
def iter_chunk_lines(
org_reports: Iterable[OrgReport],
instance_metadata: InstanceMetadata,
diff --git a/posthog/temporal/usage_report/queries.py b/posthog/temporal/usage_report/queries.py
index cbebb57d83fe..31232971cb73 100644
--- a/posthog/temporal/usage_report/queries.py
+++ b/posthog/temporal/usage_report/queries.py
@@ -69,6 +69,7 @@ from posthog.tasks.usage_report import (
get_teams_with_hog_function_calls_in_period,
get_teams_with_hog_function_fetch_calls_in_period,
get_teams_with_logs_bytes_in_period,
+ get_teams_with_logs_distinct_patterns_in_period,
get_teams_with_logs_records_in_period,
get_teams_with_logs_retention_byte_days_in_period,
get_teams_with_logs_retention_bytes_in_period,
@@ -572,6 +573,10 @@ QUERIES: list[QuerySpec] = [
name="teams_with_logs_retention_byte_days_in_period",
fn=get_teams_with_logs_retention_byte_days_in_period,
),
+ QuerySpec(
+ name="teams_with_logs_distinct_patterns_in_period",
+ fn=get_teams_with_logs_distinct_patterns_in_period,
+ ),
# ---- ClickHouse: APM tracing ---------------------------------------------
QuerySpec(
name="apm_tracing_usage",
diff --git a/posthog/temporal/usage_report/workflow.py b/posthog/temporal/usage_report/workflow.py
index ff1b2450f103..83cd98fa0d23 100644
--- a/posthog/temporal/usage_report/workflow.py
+++ b/posthog/temporal/usage_report/workflow.py
@@ -56,6 +56,8 @@ SANDBOX_COMPUTE_QUERY_NAME = "sandbox_compute_usage"
USAGE_COUNTER_SERVICE_PATCH_ID = "usage-report-counter-service-2026-09"
PARALLEL_COUNTER_QUERIES_PATCH_ID = "usage-report-parallel-counter-queries-2026-10"
AGGREGATION_QUEUE_PATCH_ID = "usage-report-aggregation-queue-2026-10"
+LOGS_DISTINCT_PATTERNS_QUERY_PATCH_ID = "usage-report-logs-distinct-patterns-query-2026-10"
+LOGS_DISTINCT_PATTERNS_QUERY_NAME = "teams_with_logs_distinct_patterns_in_period"
def _queries_for_sandbox_compute_patch(patch_applied: bool) -> list[QuerySpec]:
@@ -64,6 +66,12 @@ def _queries_for_sandbox_compute_patch(patch_applied: bool) -> list[QuerySpec]:
return [spec for spec in QUERIES if spec.name != SANDBOX_COMPUTE_QUERY_NAME]
+def _queries_for_logs_distinct_patterns_patch(queries: list[QuerySpec], patch_applied: bool) -> list[QuerySpec]:
+ if patch_applied:
+ return queries
+ return [spec for spec in queries if spec.name != LOGS_DISTINCT_PATTERNS_QUERY_NAME]
+
+
def build_context(inputs: RunUsageReportsInputs, run_id: str, now: datetime) -> WorkflowContext:
"""Compute the period and S3 layout context. Pure function so it's safe
to call from the workflow body (no real-time access).
@@ -124,6 +132,9 @@ class RunUsageReportsWorkflow(PostHogWorkflow):
try:
ctx = build_context(inputs, run_id=workflow.info().run_id, now=started_at)
queries = _queries_for_sandbox_compute_patch(workflow.patched(SANDBOX_COMPUTE_QUERY_PATCH_ID))
+ queries = _queries_for_logs_distinct_patterns_patch(
+ queries, workflow.patched(LOGS_DISTINCT_PATTERNS_QUERY_PATCH_ID)
+ )
if workflow.patched(USAGE_COUNTER_SERVICE_PATCH_ID):
ctx.usage_counter_plan = await workflow.execute_activity(
plan_usage_counters,