:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(logs): add default span and log metric rules behind a team allowlist (#126558)
Daniel Visca849c25c2da63
diff --git a/nodejs/src/logs/config.ts b/nodejs/src/logs/config.tsindex 47ffa1a313f0..a022912ba191 100644--- a/nodejs/src/logs/config.ts+++ b/nodejs/src/logs/config.ts@@ -59,6 +59,8 @@ export type LogsIngestionConsumerConfig = {LOGS_METRICS_RULES_ENABLED_TEAMS: string/** When `true`, metric-rule evaluation and emission are skipped for every team. */LOGS_METRICS_RULES_KILLSWITCH: boolean+ /** Comma-separated team IDs, or `*` for all teams, that also get the built-in default metric rules. Independent of `LOGS_METRICS_RULES_ENABLED_TEAMS`. */+ LOGS_DEFAULT_METRIC_RULES_ENABLED_TEAMS: string/** capture-logs OTLP metrics endpoint (e.g. `http://capture-logs:3308/i/v1/metrics`); empty disables emission. */LOGS_METRICS_RULES_EXPORT_URL: string/** Comma-separated team IDs, or `*` for all teams, or empty to disable per-row retention rule evaluation entirely. */@@ -133,6 +135,7 @@ export function getDefaultLogsIngestionConsumerConfig(): LogsIngestionConsumerCoLOGS_METRICS_RULES_ENABLED_TEAMS: isProdEnv() ? '' : '*',LOGS_METRICS_RULES_KILLSWITCH: false,LOGS_METRICS_RULES_EXPORT_URL: '',+ LOGS_DEFAULT_METRIC_RULES_ENABLED_TEAMS: '',// Off in prod until per-team rollout; on in dev for local end-to-end testing.LOGS_RETENTION_ENABLED_TEAMS: isProdEnv() ? '' : '*',LOGS_BACKFILL_ENABLED_TEAMS: isProdEnv() ? '' : '*',@@ -187,6 +190,8 @@ export type TracesIngestionConsumerConfig = {TRACES_METRICS_RULES_ENABLED_TEAMS: stringTRACES_METRICS_RULES_KILLSWITCH: booleanTRACES_METRICS_RULES_EXPORT_URL: string+ /** Comma-separated team IDs, or `*` for all teams, that also get the built-in default span metric rules. */+ TRACES_DEFAULT_METRIC_RULES_ENABLED_TEAMS: string/** Comma-separated team IDs, or `*` for all teams, or empty to disable per-row span retention rules. */TRACES_RETENTION_ENABLED_TEAMS: string/** When `true`, span retention rules are never evaluated (spans keep the team default via the batch header). */@@ -220,6 +225,7 @@ export function getDefaultTracesIngestionConsumerConfig(): TracesIngestionConsumTRACES_METRICS_RULES_ENABLED_TEAMS: isProdEnv() ? '' : '*',TRACES_METRICS_RULES_KILLSWITCH: false,TRACES_METRICS_RULES_EXPORT_URL: '',+ TRACES_DEFAULT_METRIC_RULES_ENABLED_TEAMS: '',// Same rollout shape as the logs retention rules: on locally, off in prod until enabled per team.TRACES_RETENTION_ENABLED_TEAMS: isProdEnv() ? '' : '*',TRACES_RETENTION_KILLSWITCH: false,diff --git a/nodejs/src/logs/logs-ingestion-consumer.test.ts b/nodejs/src/logs/logs-ingestion-consumer.test.tsindex 970a87891be8..b3f76d34d4c5 100644--- a/nodejs/src/logs/logs-ingestion-consumer.test.ts+++ b/nodejs/src/logs/logs-ingestion-consumer.test.ts@@ -2633,6 +2633,38 @@ describe('LogsIngestionConsumer', () => {expect(mockEmitter.emit).not.toHaveBeenCalled()})+ it.each([+ ['emits', {}, ['posthog.log.records']],+ ['does not emit under the killswitch', { LOGS_METRICS_RULES_KILLSWITCH: true }, null],+ ])(+ 'default rules for an allowlisted team without rules of its own: %s',+ async (_label, overrides, expectedMetricNames) => {+ // Team rules are off, so only the default-rules allowlist can turn emission on.+ await createConsumerWithMetricRules({+ LOGS_METRICS_RULES_ENABLED_TEAMS: '',+ LOGS_DEFAULT_METRIC_RULES_ENABLED_TEAMS: String(team2.id),+ ...overrides,+ })+ const messages = await createKafkaMessages([createLogMessage()], {+ token: team2.api_token,+ bytes_uncompressed: '400',+ record_count: '1',+ })+ await waitForBackgroundTasks(consumer.processKafkaBatch(messages))+ if (expectedMetricNames === null) {+ expect(mockEmitter.emit).not.toHaveBeenCalled()+ return+ }+ expect(mockEmitter.emit).toHaveBeenCalledTimes(1)+ const [, teamId, payload] = mockEmitter.emit.mock.calls[0]+ expect(teamId).toBe(team2.id)+ const names = payload.resourceMetrics.flatMap((rm: any) =>+ rm.scopeMetrics[0].metrics.map((m: any) => m.name)+ )+ expect(names).toEqual(expectedMetricNames)+ }+ )+it.each([['killswitch', { LOGS_METRICS_RULES_KILLSWITCH: true }],['team gating', { LOGS_METRICS_RULES_ENABLED_TEAMS: '' }],diff --git a/nodejs/src/logs/logs-ingestion-consumer.ts b/nodejs/src/logs/logs-ingestion-consumer.tsindex e3efe0513368..b0152f62c37a 100644--- a/nodejs/src/logs/logs-ingestion-consumer.ts+++ b/nodejs/src/logs/logs-ingestion-consumer.ts@@ -49,10 +49,11 @@ import {sniffJsonLogAttributes,} from './log-record-avro'import type { CompiledMetricRule, MetricRuleSource } from './metrics-rules/compile-metric-rules'+import { withDefaultMetricRules } from './metrics-rules/default-metric-rules'import { MetricRulesCache } from './metrics-rules/metric-rules-cache'import { LogsMetricsEmitter } from './metrics-rules/metrics-emitter'import { buildMetricRulesOtlpPayload } from './metrics-rules/otlp-payload'-import { type BatchTallies, createBatchTallies, tallyRecords } from './metrics-rules/tally'+import { type BatchTallies, MetricSeriesAdmission, createBatchTallies, tallyRecords } from './metrics-rules/tally'import { LOGS_DLQ_OUTPUT, LOGS_OUTPUT, LogsDlqOutput, LogsOutput } from './outputs/outputs'import { PatternMessageKeysCache } from './pattern-message-keys-cache'import { EMPTY_STAGE_DROP_STATS, type PipelineStage } from './pipeline/log-processing-pipeline'@@ -407,6 +408,8 @@ export class LogsIngestionConsumer {private readonly billingProrateEnabled: booleanprivate readonly metricRulesEnabledTeamsRaw: stringprivate readonly metricRulesKillswitch: boolean+ private readonly defaultMetricRulesEnabledTeamsRaw: string+ private readonly metricSeriesAdmission = new MetricSeriesAdmission()private readonly transformationsEnabledTeamsRaw: stringprivate readonly transformationsKillswitch: booleanprivate readonly retentionEnabledTeamsRaw: string@@ -465,6 +468,7 @@ export class LogsIngestionConsumer {this.billingProrateEnabled = mergedConfig.LOGS_BILLING_PRORATE_ENABLEDthis.metricRulesEnabledTeamsRaw = mergedConfig.LOGS_METRICS_RULES_ENABLED_TEAMSthis.metricRulesKillswitch = mergedConfig.LOGS_METRICS_RULES_KILLSWITCH+ this.defaultMetricRulesEnabledTeamsRaw = mergedConfig.LOGS_DEFAULT_METRIC_RULES_ENABLED_TEAMSthis.transformationsEnabledTeamsRaw = mergedConfig.LOGS_TRANSFORMATIONS_ENABLED_TEAMSthis.transformationsKillswitch = mergedConfig.LOGS_TRANSFORMATIONS_KILLSWITCHthis.retentionEnabledTeamsRaw = mergedConfig.LOGS_RETENTION_ENABLED_TEAMS@@ -496,11 +500,16 @@ export class LogsIngestionConsumer {return Promise.resolve(logsSettings.retention_days ?? DEFAULT_LOGS_RETENTION_DAYS)}+ private canEmitMetricRules(): boolean {+ return !this.metricRulesKillswitch && !!this.deps.metricRulesCache && !!this.deps.metricsEmitter+ }+private isMetricRulesEnabledForTeam(teamId: number): boolean {- if (this.metricRulesKillswitch || !this.deps.metricRulesCache || !this.deps.metricsEmitter) {- return false- }- return teamIdMatchesCsv(this.metricRulesEnabledTeamsRaw, teamId)+ return this.canEmitMetricRules() && teamIdMatchesCsv(this.metricRulesEnabledTeamsRaw, teamId)+ }++ private isDefaultMetricRulesEnabledForTeam(teamId: number): boolean {+ return this.canEmitMetricRules() && teamIdMatchesCsv(this.defaultMetricRulesEnabledTeamsRaw, teamId)}// Transformations write hog_function logs, which could come back here as internal logs and loop.@@ -969,17 +978,24 @@ export class LogsIngestionConsumer {byTeam: MetricTalliesByTeam,message: LogsIngestionMessage): Promise<{ token: string; rules: CompiledMetricRule[]; tallies: BatchTallies } | null> {- if (!this.appliesTeamLogRules(message) || !this.isMetricRulesEnabledForTeam(message.teamId)) {+ if (!this.appliesTeamLogRules(message)) {+ return null+ }+ const teamRulesEnabled = this.isMetricRulesEnabledForTeam(message.teamId)+ const defaultRulesEnabled = this.isDefaultMetricRulesEnabledForTeam(message.teamId)+ if (!teamRulesEnabled && !defaultRulesEnabled) {return null}let state = byTeam.get(message.teamId)if (!state) {- let rules: CompiledMetricRule[]+ let rules: CompiledMetricRule[] = []try {- const all = await this.deps.metricRulesCache!.getCompiledRules(message.teamId)- // Each consumer tallies only its own record source: the logs consumer runs- // `logs` rules, the traces consumer runs `spans` rules.- rules = all.filter((r) => r.source === this.metricRuleSource)+ if (teamRulesEnabled) {+ const all = await this.deps.metricRulesCache!.getCompiledRules(message.teamId)+ // Each consumer tallies only its own record source: the logs consumer runs+ // `logs` rules, the traces consumer runs `spans` rules.+ rules = all.filter((r) => r.source === this.metricRuleSource)+ }} catch (error) {// Fail open: metric rules are a purely additive side feature, so a rules-fetch// failure (e.g. a Postgres blip) must never DLQ or block the log records —@@ -990,6 +1006,9 @@ export class LogsIngestionConsumer {})return null}+ if (defaultRulesEnabled) {+ rules = withDefaultMetricRules(rules, this.metricRuleSource)+ }// Re-check after the await: concurrent messages for the same team race here,// and replacing an existing state would silently drop its accumulated tallies.state = byTeam.get(message.teamId)@@ -1085,11 +1104,19 @@ export class LogsIngestionConsumer {metricRuleState || jsonAttributeKey? (records: LogRecord[]) => {if (metricRuleState) {+ const nowMs = Date.now()tallyRecords(metricRuleState.rules,records,metricRuleState.tallies,- Date.now()+ nowMs,+ (ruleId, labelKey) =>+ this.metricSeriesAdmission.admit(+ message.teamId,+ ruleId,+ labelKey,+ nowMs+ ))}if (jsonAttributeKey) {diff --git a/nodejs/src/logs/metrics-rules/default-metric-rules.test.ts b/nodejs/src/logs/metrics-rules/default-metric-rules.test.tsnew file mode 100644index 000000000000..1308b2a96db3--- /dev/null+++ b/nodejs/src/logs/metrics-rules/default-metric-rules.test.ts@@ -0,0 +1,118 @@+import type { LogRecord } from '../log-record-avro'+import { type CompiledMetricRule } from './compile-metric-rules'+import { DEFAULT_METRIC_RULES, withDefaultMetricRules } from './default-metric-rules'+import { createBatchTallies, tallyRecords } from './tally'++const NOW_MS = 1_700_000_000_000++const spanRecord = (overrides: Record<string, unknown> = {}): LogRecord =>+ ({+ uuid: null,+ trace_id: null,+ span_id: null,+ trace_flags: null,+ timestamp: NOW_MS * 1000,+ end_time: NOW_MS * 1000 + 40_000, // 40ms+ observed_timestamp: null,+ body: null,+ severity_text: null,+ severity_number: null,+ service_name: 'api',+ resource_attributes: {},+ instrumentation_scope: null,+ event_name: null,+ attributes: {},+ name: 'GET /checkout',+ kind: 2, // SERVER+ status_code: 1, // OK+ status_message: null,+ ...overrides,+ }) as unknown as LogRecord++const logRecord = (overrides: Partial<LogRecord> = {}): LogRecord => ({+ uuid: null,+ trace_id: null,+ span_id: null,+ trace_flags: null,+ timestamp: NOW_MS * 1000,+ observed_timestamp: null,+ body: 'hello',+ severity_text: 'ERROR',+ severity_number: 17,+ service_name: 'api',+ resource_attributes: {},+ instrumentation_scope: null,+ event_name: null,+ attributes: {},+ ...overrides,+})++function tallyDefaults(source: 'logs' | 'spans', records: LogRecord[]): Map<string, number> {+ const rules = DEFAULT_METRIC_RULES[source]+ const tallies = createBatchTallies()+ tallyRecords(rules, records, tallies, NOW_MS)+ const counts = new Map<string, number>()+ for (const entry of tallies.byRule.get(rules[0]!.id)?.values() ?? []) {+ counts.set(entry.labelValues.join('|'), entry.count)+ }+ return counts+}++describe('default metric rules', () => {+ it('names every default metric under the reserved posthog. prefix so it cannot clash with a team rule', () => {+ for (const rule of [...DEFAULT_METRIC_RULES.spans, ...DEFAULT_METRIC_RULES.logs]) {+ expect(rule.metricName.startsWith('posthog.')).toBe(true)+ }+ })++ it('tallies server, client, producer and consumer spans by service, kind, name and status', () => {+ const counts = tallyDefaults('spans', [+ spanRecord(),+ spanRecord({ status_code: 2 }),+ spanRecord({ kind: 3, name: 'SELECT orders' }),+ spanRecord({ kind: 4, name: 'publish events' }),+ spanRecord({ kind: 5, name: 'consume events' }),+ ])+ expect(Object.fromEntries(counts)).toEqual({+ 'api|SERVER|GET /checkout|OK': 1,+ 'api|SERVER|GET /checkout|ERROR': 1,+ 'api|CLIENT|SELECT orders|OK': 1,+ 'api|PRODUCER|publish events|OK': 1,+ 'api|CONSUMER|consume events|OK': 1,+ })+ })++ it('skips internal spans, whose names are the most numerous and least useful as series', () => {+ expect(tallyDefaults('spans', [spanRecord({ kind: 1, name: 'render' })]).size).toBe(0)+ })++ it('counts every log record by service and severity', () => {+ const counts = tallyDefaults('logs', [+ logRecord(),+ logRecord(),+ logRecord({ severity_text: 'INFO' }),+ logRecord({ service_name: 'web', severity_text: 'INFO' }),+ ])+ expect(Object.fromEntries(counts)).toEqual({ 'api|ERROR': 2, 'api|INFO': 1, 'web|INFO': 1 })+ })++ describe('withDefaultMetricRules', () => {+ const userRule: CompiledMetricRule = {+ id: 'user-rule',+ metricName: 'checkout.errors',+ source: 'logs',+ filterGroup: null,+ valueAttribute: null,+ groupBy: [],+ }++ it('appends the defaults for the source after the team rules', () => {+ expect(withDefaultMetricRules([userRule], 'logs')).toEqual([userRule, ...DEFAULT_METRIC_RULES.logs])+ })++ it('lets a team rule with the same metric name win over the default', () => {+ const shadowing = { ...userRule, metricName: DEFAULT_METRIC_RULES.logs[0]!.metricName }+ expect(withDefaultMetricRules([shadowing], 'logs')).toEqual([shadowing])+ })+ })+})diff --git a/nodejs/src/logs/metrics-rules/default-metric-rules.ts b/nodejs/src/logs/metrics-rules/default-metric-rules.tsnew file mode 100644index 000000000000..f8fb813303fd--- /dev/null+++ b/nodejs/src/logs/metrics-rules/default-metric-rules.ts@@ -0,0 +1,54 @@+import {+ type CompiledMetricRule,+ type MetricRuleRow,+ type MetricRuleSource,+ compileMetricRules,+} from './compile-metric-rules'+import { SPAN_VALUE_DURATION_MS } from './tally'++const SPAN_KIND_INTERNAL = '1'++/** Built-in rules, not stored rows: the consumer adds them for teams in the default-rules allowlist. */+const DEFAULT_METRIC_RULE_ROWS: MetricRuleRow[] = [+ {+ // INTERNAL spans are the most numerous, and their names are the most likely to carry ids.+ id: 'default:spans:duration',+ metric_name: 'posthog.span.duration',+ source: 'spans',+ filter_group: {+ type: 'AND',+ values: [+ {+ type: 'AND',+ values: [{ key: 'kind', operator: 'is_not', value: SPAN_KIND_INTERNAL, type: 'span_attribute' }],+ },+ ],+ },+ value_attribute: SPAN_VALUE_DURATION_MS,+ group_by: ['service_name', 'kind', 'name', 'status_code'],+ },+ {+ id: 'default:logs:records',+ metric_name: 'posthog.log.records',+ source: 'logs',+ filter_group: null,+ value_attribute: null,+ group_by: ['service_name', 'severity_text'],+ },+]++const compiledDefaults = compileMetricRules(DEFAULT_METRIC_RULE_ROWS)++export const DEFAULT_METRIC_RULES: Record<MetricRuleSource, CompiledMetricRule[]> = {+ logs: compiledDefaults.filter((rule) => rule.source === 'logs'),+ spans: compiledDefaults.filter((rule) => rule.source === 'spans'),+}++/** Appends the source's default rules; a team rule with the same metric name replaces the default. */+export function withDefaultMetricRules(+ teamRules: CompiledMetricRule[],+ source: MetricRuleSource+): CompiledMetricRule[] {+ const teamMetricNames = new Set(teamRules.map((rule) => rule.metricName))+ return [...teamRules, ...DEFAULT_METRIC_RULES[source].filter((rule) => !teamMetricNames.has(rule.metricName))]+}diff --git a/nodejs/src/logs/metrics-rules/metric-rules.test.ts b/nodejs/src/logs/metrics-rules/metric-rules.test.tsindex 25108eed3a7d..2f35d6451849 100644--- a/nodejs/src/logs/metrics-rules/metric-rules.test.ts+++ b/nodejs/src/logs/metrics-rules/metric-rules.test.ts@@ -1,6 +1,14 @@import type { LogRecord } from '../log-record-avro'import { type MetricRuleRow, compileMetricRules } from './compile-metric-rules'-import { MAX_LABEL_SETS_PER_RULE, MAX_LABEL_VALUE_LENGTH, createBatchTallies, tallyRecords } from './tally'+import {+ DEFAULT_HISTOGRAM_BOUNDS,+ MAX_LABEL_SETS_PER_RULE,+ MAX_LABEL_VALUE_LENGTH,+ MetricSeriesAdmission,+ OVERFLOW_LABEL_KEY,+ createBatchTallies,+ tallyRecords,+} from './tally'const NOW_MS = 1_700_000_000_000@@ -167,17 +175,100 @@ describe('metric rules compile + tally', () => {expect(entries[0]!.count).toBe(2)})- it('caps distinct label sets per rule and counts the overflow', () => {+ it('folds label sets past the cap into one overflow series instead of dropping them', () => {const rules = compileMetricRules([ruleRow({ filter_group: null, group_by: ['attributes.request_id'] })])const tallies = createBatchTallies()const records = Array.from({ length: MAX_LABEL_SETS_PER_RULE + 3 }, (_, i) =>record({ attributes: { request_id: `req-${i}` } }))tallyRecords(rules, records, tallies, NOW_MS)- expect(tallies.byRule.get('rule-1')!.size).toBe(MAX_LABEL_SETS_PER_RULE)+ const ruleTallies = tallies.byRule.get('rule-1')!+ expect(ruleTallies.size).toBe(MAX_LABEL_SETS_PER_RULE + 1)+ const overflow = ruleTallies.get(OVERFLOW_LABEL_KEY)!+ expect(overflow.overflow).toBe(true)+ expect(overflow.count).toBe(3)expect(tallies.seriesOverflow.get('rule-1')).toBe(3)})+ it('keeps known label sets on their own series after the cap is reached', () => {+ const rules = compileMetricRules([ruleRow({ filter_group: null, group_by: ['attributes.request_id'] })])+ const tallies = createBatchTallies()+ const records = Array.from({ length: MAX_LABEL_SETS_PER_RULE + 1 }, (_, i) =>+ record({ attributes: { request_id: `req-${i}` } })+ )+ records.push(record({ attributes: { request_id: 'req-0' } }))+ tallyRecords(rules, records, tallies, NOW_MS)+ const ruleTallies = tallies.byRule.get('rule-1')!+ expect(ruleTallies.get(JSON.stringify(['req-0']))!.count).toBe(2)+ expect(ruleTallies.get(OVERFLOW_LABEL_KEY)!.count).toBe(1)+ })++ it('bounds label sets per rule across batches until the hour changes', () => {+ const rules = compileMetricRules([ruleRow({ filter_group: null, group_by: ['attributes.request_id'] })])+ const admission = new MetricSeriesAdmission()+ const tallyBatch = (ids: string[], nowMs: number): Map<string, { count: number }> => {+ const tallies = createBatchTallies()+ const records = ids.map((id) => record({ attributes: { request_id: id }, timestamp: nowMs * 1000 }))+ tallyRecords(rules, records, tallies, nowMs, (ruleId, labelKey) =>+ admission.admit(1, ruleId, labelKey, nowMs)+ )+ return tallies.byRule.get('rule-1')!+ }+ tallyBatch(+ Array.from({ length: MAX_LABEL_SETS_PER_RULE }, (_, i) => `req-${i}`),+ NOW_MS+ )++ const sameHour = tallyBatch(['req-0', 'req-new'], NOW_MS + 1000)+ expect(sameHour.get(JSON.stringify(['req-0']))!.count).toBe(1)+ expect(sameHour.get(JSON.stringify(['req-new']))).toBeUndefined()+ expect(sameHour.get(OVERFLOW_LABEL_KEY)!.count).toBe(1)++ const nextHour = tallyBatch(['req-new'], NOW_MS + 60 * 60 * 1000)+ expect(nextHour.get(JSON.stringify(['req-new']))!.count).toBe(1)+ })++ it('counts colliding label sets separately, so crafted FNV-1a collisions cannot pass the cap', () => {+ const admission = new MetricSeriesAdmission(1)+ expect(admission.admit(1, 'rule-a', 'costarring', NOW_MS)).toBe(true)+ expect(admission.admit(1, 'rule-a', 'liquid', NOW_MS)).toBe(false)+ })++ it('caps a team across all its rules, so rule churn cannot take the budget of other teams', () => {+ const admission = new MetricSeriesAdmission(2, 3, 10)+ expect(admission.admit(1, 'rule-a', 'x', NOW_MS)).toBe(true)+ expect(admission.admit(1, 'rule-a', 'y', NOW_MS)).toBe(true)+ expect(admission.admit(1, 'rule-b', 'x', NOW_MS)).toBe(true)+ expect(admission.admit(1, 'rule-c', 'x', NOW_MS)).toBe(false)+ expect(admission.admit(1, 'rule-a', 'x', NOW_MS)).toBe(true)+ expect(admission.admit(2, 'rule-a', 'x', NOW_MS)).toBe(true)+ })++ it('stops admitting new label sets when the process-wide budget is spent', () => {+ const admission = new MetricSeriesAdmission(2, 2, 3)+ expect(admission.admit(1, 'rule-a', 'x', NOW_MS)).toBe(true)+ expect(admission.admit(1, 'rule-a', 'y', NOW_MS)).toBe(true)+ expect(admission.admit(2, 'rule-a', 'x', NOW_MS)).toBe(true)+ expect(admission.admit(2, 'rule-b', 'z', NOW_MS)).toBe(false)+ expect(admission.admit(2, 'rule-b', 'z', NOW_MS + 60 * 60 * 1000)).toBe(true)+ })++ it.each([+ [0, 0],+ [5, 1],+ [5.01, 2],+ [10_000, DEFAULT_HISTOGRAM_BOUNDS.length - 1],+ [10_001, DEFAULT_HISTOGRAM_BOUNDS.length],+ ])('puts value %p in histogram bucket %p (upper bound inclusive)', (value, bucket) => {+ const rules = compileMetricRules([ruleRow({ filter_group: null, value_attribute: 'attributes.v' })])+ const tallies = createBatchTallies()+ tallyRecords(rules, [record({ attributes: { v: String(value) } })], tallies, NOW_MS)+ const entry = [...tallies.byRule.get('rule-1')!.values()][0]!+ const expected = new Array(DEFAULT_HISTOGRAM_BOUNDS.length + 1).fill(0)+ expected[bucket] = 1+ expect(entry.bucketCounts).toEqual(expected)+ })+it('captures an exemplar trace id from a matching record and ignores zeroed trace ids', () => {const rules = compileMetricRules([ruleRow({ filter_group: null })])const tallies = createBatchTallies()diff --git a/nodejs/src/logs/metrics-rules/otlp-payload.test.ts b/nodejs/src/logs/metrics-rules/otlp-payload.test.tsindex dc26419b4964..d57d25f0efa1 100644--- a/nodejs/src/logs/metrics-rules/otlp-payload.test.ts+++ b/nodejs/src/logs/metrics-rules/otlp-payload.test.ts@@ -1,7 +1,7 @@import type { LogRecord } from '../log-record-avro'import { compileMetricRules } from './compile-metric-rules'import { buildMetricRulesOtlpPayload } from './otlp-payload'-import { createBatchTallies, tallyRecords } from './tally'+import { DEFAULT_HISTOGRAM_BOUNDS, MAX_LABEL_SETS_PER_RULE, createBatchTallies, tallyRecords } from './tally'const NOW_MS = 1_700_000_000_123// Snapped down to the enclosing 10s bucket, in nanos.@@ -69,7 +69,7 @@ describe('buildMetricRulesOtlpPayload', () => {expect(errorPoint.timeUnixNano).toBe(SNAPPED_NANOS)})- it('emits a value-attribute rule as a delta histogram carrying count and sum', () => {+ it('emits a value-attribute rule as a delta histogram with explicit bucket counts', () => {const payload = build([distributionRule], [record(), record({ attributes: { duration_ms: '7.5' } })])const metric = payload.resourceMetrics[0].scopeMetrics[0].metrics[0]expect(metric.name).toBe('log.request_duration')@@ -77,9 +77,31 @@ describe('buildMetricRulesOtlpPayload', () => {const dp = metric.histogram.dataPoints[0]expect(dp.count).toBe(2)expect(dp.sum).toBe(20)- expect(dp.explicitBounds).toEqual([])- expect(dp.bucketCounts).toEqual([2])+ expect(dp.explicitBounds).toEqual(DEFAULT_HISTOGRAM_BOUNDS)+ const expectedCounts = new Array(DEFAULT_HISTOGRAM_BOUNDS.length + 1).fill(0)+ expectedCounts[2] = 1 // 7.5 falls in (5, 10]+ expectedCounts[3] = 1 // 12.5 falls in (10, 25]+ expect(dp.bucketCounts).toEqual(expectedCounts)+ expect(dp.attributes).toEqual([{ key: 'posthog.bucket_layout', value: { stringValue: 'otel_default_v1' } }])expect(dp.timeUnixNano).toBe(SNAPPED_NANOS)+ expect(metric.unit).toBeUndefined()+ })++ it('emits a span duration rule with the ms unit', () => {+ const span = { ...record(), end_time: NOW_MS * 1000 + 40_000 } as LogRecord+ const payload = build([{ ...distributionRule, source: 'spans', value_attribute: 'duration_ms' }], [span])+ expect(payload.resourceMetrics[0].scopeMetrics[0].metrics[0].unit).toBe('ms')+ })++ it('emits the overflow series with only the otel.metric.overflow attribute and no service', () => {+ const records = Array.from({ length: MAX_LABEL_SETS_PER_RULE + 2 }, (_, i) =>+ record({ service_name: `svc-${i}` })+ )+ const payload = build([{ ...countRule, group_by: ['service_name', 'severity_text'] }], records)+ const overflowResource = payload.resourceMetrics.find((rm: any) => rm.resource.attributes.length === 0)+ const dp = overflowResource.scopeMetrics[0].metrics[0].sum.dataPoints[0]+ expect(dp.attributes).toEqual([{ key: 'otel.metric.overflow', value: { boolValue: true } }])+ expect(dp.asDouble).toBe(2)})it('strips the attributes. prefix from group-by label names', () => {diff --git a/nodejs/src/logs/metrics-rules/otlp-payload.ts b/nodejs/src/logs/metrics-rules/otlp-payload.tsindex 4858def399a3..73a26f525197 100644--- a/nodejs/src/logs/metrics-rules/otlp-payload.ts+++ b/nodejs/src/logs/metrics-rules/otlp-payload.ts@@ -1,5 +1,5 @@import type { CompiledMetricRule } from './compile-metric-rules'-import type { BatchTallies, MetricTallyEntry } from './tally'+import { type BatchTallies, DEFAULT_HISTOGRAM_BOUNDS, type MetricTallyEntry, SPAN_VALUE_DURATION_MS } from './tally'/*** Hand-built OTLP/JSON `ExportMetricsServiceRequest` for log-generated metrics.@@ -20,6 +20,15 @@ const OTLP_TEMPORALITY_DELTA = 1const ATTRIBUTE_PREFIXES = ['attributes.', 'resource_attributes.']+/** OTel SDK convention for the series that collects data points past a cardinality limit. */+const OVERFLOW_ATTRIBUTE = { key: 'otel.metric.overflow', value: { boolValue: true } }++/**+ * Series identity does not include bucket bounds, and samples keep one bounds array per series-hour.+ * This label gives each bucket layout its own series, so samples with different layouts never merge.+ */+const BUCKET_LAYOUT_ATTRIBUTE = { key: 'posthog.bucket_layout', value: { stringValue: 'otel_default_v1' } }+/** Label name as it appears on the emitted series: map keys lose their prefix (Datadog drops `@` the same way). */function labelName(groupByKey: string): string {for (const prefix of ATTRIBUTE_PREFIXES) {@@ -53,7 +62,7 @@ export function buildMetricRulesOtlpPayload(}const serviceIndex = rule.groupBy.indexOf('service_name')for (const entry of ruleTallies.values()) {- const service = serviceIndex >= 0 ? (entry.labelValues[serviceIndex] ?? '') : ''+ const service = serviceIndex >= 0 && !entry.overflow ? (entry.labelValues[serviceIndex] ?? '') : ''let ruleMap = byService.get(service)if (!ruleMap) {ruleMap = new Map()@@ -101,15 +110,14 @@ function serializeRuleMetric(if (rule.valueAttribute) {return {name: rule.metricName,+ ...(rule.valueAttribute === SPAN_VALUE_DURATION_MS ? { unit: 'ms' } : {}),histogram: {dataPoints: entries.map((entry) => ({- ...dataPointBase(rule, entry, startTimeUnixNano, timeUnixNano),+ ...dataPointBase(rule, entry, startTimeUnixNano, timeUnixNano, BUCKET_LAYOUT_ATTRIBUTE),count: entry.count,sum: entry.sum,- // No percentile buckets yet: a single catch-all bucket keeps the shape a- // valid OTLP histogram while carrying only count + sum.- bucketCounts: [entry.count],- explicitBounds: [],+ bucketCounts: entry.bucketCounts,+ explicitBounds: DEFAULT_HISTOGRAM_BOUNDS,})),aggregationTemporality: OTLP_TEMPORALITY_DELTA,},@@ -132,10 +140,14 @@ function dataPointBase(rule: CompiledMetricRule,entry: MetricTallyEntry,startTimeUnixNano: string,- timeUnixNano: string+ timeUnixNano: string,+ extraAttribute?: Record<string, unknown>): Record<string, unknown> {- const attributes: Record<string, unknown>[] = []- for (let i = 0; i < rule.groupBy.length; i++) {+ const attributes: Record<string, unknown>[] = entry.overflow ? [OVERFLOW_ATTRIBUTE] : []+ if (extraAttribute) {+ attributes.push(extraAttribute)+ }+ for (let i = 0; i < rule.groupBy.length && !entry.overflow; i++) {const key = rule.groupBy[i]!if (key === 'service_name') {continue // carried on the resource block insteaddiff --git a/nodejs/src/logs/metrics-rules/tally.ts b/nodejs/src/logs/metrics-rules/tally.tsindex 6e0fa34db1ca..962ea524d9eb 100644--- a/nodejs/src/logs/metrics-rules/tally.ts+++ b/nodejs/src/logs/metrics-rules/tally.ts@@ -1,3 +1,5 @@+import { createHmac, randomBytes } from 'crypto'+import { decodeLogAttributeValue } from '../attribute-value'import type { LogRecord } from '../log-record-avro'import { matchFilterGroup } from '../sampling/filter-group-match'@@ -6,6 +8,14 @@ import type { CompiledMetricRule } from './compile-metric-rules'/** Cap on distinct group-by label sets per rule per batch — bounds emitted series cardinality. */export const MAX_LABEL_SETS_PER_RULE = 1000+/** Not valid JSON, so no real label set (always a JSON array) can collide with it. */+export const OVERFLOW_LABEL_KEY = '__overflow__'++/** The OTel SDK default explicit bucket bounds, so percentiles match an SDK-sent histogram. */+export const DEFAULT_HISTOGRAM_BOUNDS: readonly number[] = [+ 0, 5, 10, 25, 50, 75, 100, 250, 500, 750, 1000, 2500, 5000, 7500, 10000,+]+export const MAX_LABEL_VALUE_LENGTH = 256/**@@ -34,13 +44,15 @@ export type MetricTallyEntry = {labelValues: string[]count: numbersum: number+ bucketCounts: number[] | null+ overflow: booleanexemplarTraceId: string | nullexemplarSpanId: string | null}export type BatchTallies = {byRule: Map<string, Map<string, MetricTallyEntry>>- /** Records that matched but were dropped because the rule hit MAX_LABEL_SETS_PER_RULE. */+ /** Records that matched after the rule hit its label-set cap, tallied into the overflow series. */seriesOverflow: Map<string, number>/** Records that matched a value-attribute rule but carried a missing/non-numeric value. */valueSkipped: number@@ -114,6 +126,93 @@ function resolveLabelValue(key: string, record: LogRecord): string {return value.length > MAX_LABEL_VALUE_LENGTH ? value.slice(0, MAX_LABEL_VALUE_LENGTH) : value}+const SERIES_WINDOW_MS = 60 * 60 * 1000++export const MAX_ADMITTED_LABEL_SETS_PER_TEAM = 5_000+export const MAX_ADMITTED_LABEL_SETS = 500_000++/**+ * HMAC with a random per-process key, truncated to 48 bits. A team cannot craft colliding labels+ * offline, so a collision cannot admit extra label sets past the cap.+ */+const LABEL_HASH_KEY = randomBytes(32)++function hashLabelKey(labelKey: string): number {+ return createHmac('sha256', LABEL_HASH_KEY).update(labelKey).digest().readUIntBE(0, 6)+}++/**+ * Bounds the label sets each rule admits per hour in this process. Batch tallies start empty,+ * so without this a rule whose labels change every batch would store new series without limit.+ * The window matches the hourly series grain of metric samples.+ */+export class MetricSeriesAdmission {+ private window = -1+ private admitted = new Map<string, Set<number>>()+ // Counts survive rule deletion until the hour changes, so rule churn cannot reset a team's budget.+ private admittedByTeam = new Map<number, number>()+ private admittedTotal = 0++ constructor(+ private readonly maxPerRule: number = MAX_LABEL_SETS_PER_RULE,+ private readonly maxPerTeam: number = MAX_ADMITTED_LABEL_SETS_PER_TEAM,+ private readonly maxTotal: number = MAX_ADMITTED_LABEL_SETS+ ) {}++ public admit(teamId: number, ruleId: string, labelKey: string, nowMs: number): boolean {+ const window = Math.floor(nowMs / SERIES_WINDOW_MS)+ if (window !== this.window) {+ this.admitted.clear()+ this.admittedByTeam.clear()+ this.admittedTotal = 0+ this.window = window+ }+ const key = `${teamId}:${ruleId}`+ const labelHash = hashLabelKey(labelKey)+ let labelHashes = this.admitted.get(key)+ if (labelHashes?.has(labelHash)) {+ return true+ }+ const teamAdmitted = this.admittedByTeam.get(teamId) ?? 0+ if (+ (labelHashes?.size ?? 0) >= this.maxPerRule ||+ teamAdmitted >= this.maxPerTeam ||+ this.admittedTotal >= this.maxTotal+ ) {+ return false+ }+ if (!labelHashes) {+ labelHashes = new Set()+ this.admitted.set(key, labelHashes)+ }+ labelHashes.add(labelHash)+ this.admittedByTeam.set(teamId, teamAdmitted + 1)+ this.admittedTotal+++ return true+ }+}++function histogramBucketIndex(value: number): number {+ for (let i = 0; i < DEFAULT_HISTOGRAM_BOUNDS.length; i++) {+ if (value <= DEFAULT_HISTOGRAM_BOUNDS[i]!) {+ return i+ }+ }+ return DEFAULT_HISTOGRAM_BOUNDS.length+}++function newTallyEntry(labelValues: string[], withBuckets: boolean, overflow: boolean): MetricTallyEntry {+ return {+ labelValues,+ count: 0,+ sum: 0,+ bucketCounts: withBuckets ? new Array<number>(DEFAULT_HISTOGRAM_BOUNDS.length + 1).fill(0) : null,+ overflow,+ exemplarTraceId: null,+ exemplarSpanId: null,+ }+}+function resolveNumericValue(key: string, record: LogRecord): number | null {// Span wall-clock duration from Avro timestamp-micros, no attribute lookup.if (key === SPAN_VALUE_DURATION_MS) {@@ -166,7 +265,8 @@ export function tallyRecords(rules: CompiledMetricRule[],records: LogRecord[],tallies: BatchTallies,- nowMs: number+ nowMs: number,+ admitLabelSet?: (ruleId: string, labelKey: string) => boolean): void {if (rules.length === 0 || records.length === 0) {return@@ -207,7 +307,7 @@ export function tallyRecords(}const labelValues = rule.groupBy.map((key) => resolveLabelValue(key, record))- const labelKey = JSON.stringify(labelValues)+ let labelKey = JSON.stringify(labelValues)let ruleTallies = tallies.byRule.get(rule.id)if (!ruleTallies) {@@ -216,15 +316,32 @@ export function tallyRecords(}let entry = ruleTallies.get(labelKey)if (!entry) {- if (ruleTallies.size >= MAX_LABEL_SETS_PER_RULE) {+ const overflowEntry = ruleTallies.get(OVERFLOW_LABEL_KEY)+ const realLabelSets = ruleTallies.size - (overflowEntry ? 1 : 0)+ const admitted = realLabelSets < MAX_LABEL_SETS_PER_RULE && (admitLabelSet?.(rule.id, labelKey) ?? true)+ if (!admitted) {+ // Past the cap the record still counts, on one shared series, so rule totals stay correct.tallies.seriesOverflow.set(rule.id, (tallies.seriesOverflow.get(rule.id) ?? 0) + 1)- continue+ labelKey = OVERFLOW_LABEL_KEY+ entry = overflowEntry+ if (!entry) {+ entry = newTallyEntry(+ rule.groupBy.map(() => ''),+ rule.valueAttribute !== null,+ true+ )+ ruleTallies.set(labelKey, entry)+ }+ } else {+ entry = newTallyEntry(labelValues, rule.valueAttribute !== null, false)+ ruleTallies.set(labelKey, entry)}- entry = { labelValues, count: 0, sum: 0, exemplarTraceId: null, exemplarSpanId: null }- ruleTallies.set(labelKey, entry)}entry.count += 1entry.sum += value+ if (entry.bucketCounts) {+ entry.bucketCounts[histogramBucketIndex(value)]! += 1+ }if (!entry.exemplarTraceId) {const traceIdHex = idToHex(record.trace_id, 16)if (traceIdHex) {diff --git a/nodejs/src/logs/traces-ingestion-consumer.ts b/nodejs/src/logs/traces-ingestion-consumer.tsindex cae16c23e4de..437e58f57591 100644--- a/nodejs/src/logs/traces-ingestion-consumer.ts+++ b/nodejs/src/logs/traces-ingestion-consumer.ts@@ -50,6 +50,7 @@ export class TracesIngestionConsumer extends LogsIngestionConsumer {LOGS_METRICS_RULES_ENABLED_TEAMS: config.TRACES_METRICS_RULES_ENABLED_TEAMS,LOGS_METRICS_RULES_KILLSWITCH: config.TRACES_METRICS_RULES_KILLSWITCH,LOGS_METRICS_RULES_EXPORT_URL: config.TRACES_METRICS_RULES_EXPORT_URL,+ LOGS_DEFAULT_METRIC_RULES_ENABLED_TEAMS: config.TRACES_DEFAULT_METRIC_RULES_ENABLED_TEAMS,// Span retention gates on traces-specific env config so it rolls out// independently of log retention.LOGS_RETENTION_ENABLED_TEAMS: config.TRACES_RETENTION_ENABLED_TEAMS,