Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(logs): add default span and log metric rules behind a team allowlist (#126558)

Daniel Visca849c25c2da63
diff --git a/nodejs/src/logs/config.ts b/nodejs/src/logs/config.ts
index 47ffa1a313f0..a022912ba191 100644
--- a/nodejs/src/logs/config.ts
+++ b/nodejs/src/logs/config.ts
@@ -59,6 +59,8 @@ export type LogsIngestionConsumerConfig = {
LOGS_METRICS_RULES_ENABLED_TEAMS: string
/** When `true`, metric-rule evaluation and emission are skipped for every team. */
LOGS_METRICS_RULES_KILLSWITCH: boolean
+ /** Comma-separated team IDs, or `*` for all teams, that also get the built-in default metric rules. Independent of `LOGS_METRICS_RULES_ENABLED_TEAMS`. */
+ LOGS_DEFAULT_METRIC_RULES_ENABLED_TEAMS: string
/** capture-logs OTLP metrics endpoint (e.g. `http://capture-logs:3308/i/v1/metrics`); empty disables emission. */
LOGS_METRICS_RULES_EXPORT_URL: string
/** Comma-separated team IDs, or `*` for all teams, or empty to disable per-row retention rule evaluation entirely. */
@@ -133,6 +135,7 @@ export function getDefaultLogsIngestionConsumerConfig(): LogsIngestionConsumerCo
LOGS_METRICS_RULES_ENABLED_TEAMS: isProdEnv() ? '' : '*',
LOGS_METRICS_RULES_KILLSWITCH: false,
LOGS_METRICS_RULES_EXPORT_URL: '',
+ LOGS_DEFAULT_METRIC_RULES_ENABLED_TEAMS: '',
// Off in prod until per-team rollout; on in dev for local end-to-end testing.
LOGS_RETENTION_ENABLED_TEAMS: isProdEnv() ? '' : '*',
LOGS_BACKFILL_ENABLED_TEAMS: isProdEnv() ? '' : '*',
@@ -187,6 +190,8 @@ export type TracesIngestionConsumerConfig = {
TRACES_METRICS_RULES_ENABLED_TEAMS: string
TRACES_METRICS_RULES_KILLSWITCH: boolean
TRACES_METRICS_RULES_EXPORT_URL: string
+ /** Comma-separated team IDs, or `*` for all teams, that also get the built-in default span metric rules. */
+ TRACES_DEFAULT_METRIC_RULES_ENABLED_TEAMS: string
/** Comma-separated team IDs, or `*` for all teams, or empty to disable per-row span retention rules. */
TRACES_RETENTION_ENABLED_TEAMS: string
/** When `true`, span retention rules are never evaluated (spans keep the team default via the batch header). */
@@ -220,6 +225,7 @@ export function getDefaultTracesIngestionConsumerConfig(): TracesIngestionConsum
TRACES_METRICS_RULES_ENABLED_TEAMS: isProdEnv() ? '' : '*',
TRACES_METRICS_RULES_KILLSWITCH: false,
TRACES_METRICS_RULES_EXPORT_URL: '',
+ TRACES_DEFAULT_METRIC_RULES_ENABLED_TEAMS: '',
// Same rollout shape as the logs retention rules: on locally, off in prod until enabled per team.
TRACES_RETENTION_ENABLED_TEAMS: isProdEnv() ? '' : '*',
TRACES_RETENTION_KILLSWITCH: false,
diff --git a/nodejs/src/logs/logs-ingestion-consumer.test.ts b/nodejs/src/logs/logs-ingestion-consumer.test.ts
index 970a87891be8..b3f76d34d4c5 100644
--- a/nodejs/src/logs/logs-ingestion-consumer.test.ts
+++ b/nodejs/src/logs/logs-ingestion-consumer.test.ts
@@ -2633,6 +2633,38 @@ describe('LogsIngestionConsumer', () => {
expect(mockEmitter.emit).not.toHaveBeenCalled()
})
+ it.each([
+ ['emits', {}, ['posthog.log.records']],
+ ['does not emit under the killswitch', { LOGS_METRICS_RULES_KILLSWITCH: true }, null],
+ ])(
+ 'default rules for an allowlisted team without rules of its own: %s',
+ async (_label, overrides, expectedMetricNames) => {
+ // Team rules are off, so only the default-rules allowlist can turn emission on.
+ await createConsumerWithMetricRules({
+ LOGS_METRICS_RULES_ENABLED_TEAMS: '',
+ LOGS_DEFAULT_METRIC_RULES_ENABLED_TEAMS: String(team2.id),
+ ...overrides,
+ })
+ const messages = await createKafkaMessages([createLogMessage()], {
+ token: team2.api_token,
+ bytes_uncompressed: '400',
+ record_count: '1',
+ })
+ await waitForBackgroundTasks(consumer.processKafkaBatch(messages))
+ if (expectedMetricNames === null) {
+ expect(mockEmitter.emit).not.toHaveBeenCalled()
+ return
+ }
+ expect(mockEmitter.emit).toHaveBeenCalledTimes(1)
+ const [, teamId, payload] = mockEmitter.emit.mock.calls[0]
+ expect(teamId).toBe(team2.id)
+ const names = payload.resourceMetrics.flatMap((rm: any) =>
+ rm.scopeMetrics[0].metrics.map((m: any) => m.name)
+ )
+ expect(names).toEqual(expectedMetricNames)
+ }
+ )
+
it.each([
['killswitch', { LOGS_METRICS_RULES_KILLSWITCH: true }],
['team gating', { LOGS_METRICS_RULES_ENABLED_TEAMS: '' }],
diff --git a/nodejs/src/logs/logs-ingestion-consumer.ts b/nodejs/src/logs/logs-ingestion-consumer.ts
index e3efe0513368..b0152f62c37a 100644
--- a/nodejs/src/logs/logs-ingestion-consumer.ts
+++ b/nodejs/src/logs/logs-ingestion-consumer.ts
@@ -49,10 +49,11 @@ import {
sniffJsonLogAttributes,
} from './log-record-avro'
import type { CompiledMetricRule, MetricRuleSource } from './metrics-rules/compile-metric-rules'
+import { withDefaultMetricRules } from './metrics-rules/default-metric-rules'
import { MetricRulesCache } from './metrics-rules/metric-rules-cache'
import { LogsMetricsEmitter } from './metrics-rules/metrics-emitter'
import { buildMetricRulesOtlpPayload } from './metrics-rules/otlp-payload'
-import { type BatchTallies, createBatchTallies, tallyRecords } from './metrics-rules/tally'
+import { type BatchTallies, MetricSeriesAdmission, createBatchTallies, tallyRecords } from './metrics-rules/tally'
import { LOGS_DLQ_OUTPUT, LOGS_OUTPUT, LogsDlqOutput, LogsOutput } from './outputs/outputs'
import { PatternMessageKeysCache } from './pattern-message-keys-cache'
import { EMPTY_STAGE_DROP_STATS, type PipelineStage } from './pipeline/log-processing-pipeline'
@@ -407,6 +408,8 @@ export class LogsIngestionConsumer {
private readonly billingProrateEnabled: boolean
private readonly metricRulesEnabledTeamsRaw: string
private readonly metricRulesKillswitch: boolean
+ private readonly defaultMetricRulesEnabledTeamsRaw: string
+ private readonly metricSeriesAdmission = new MetricSeriesAdmission()
private readonly transformationsEnabledTeamsRaw: string
private readonly transformationsKillswitch: boolean
private readonly retentionEnabledTeamsRaw: string
@@ -465,6 +468,7 @@ export class LogsIngestionConsumer {
this.billingProrateEnabled = mergedConfig.LOGS_BILLING_PRORATE_ENABLED
this.metricRulesEnabledTeamsRaw = mergedConfig.LOGS_METRICS_RULES_ENABLED_TEAMS
this.metricRulesKillswitch = mergedConfig.LOGS_METRICS_RULES_KILLSWITCH
+ this.defaultMetricRulesEnabledTeamsRaw = mergedConfig.LOGS_DEFAULT_METRIC_RULES_ENABLED_TEAMS
this.transformationsEnabledTeamsRaw = mergedConfig.LOGS_TRANSFORMATIONS_ENABLED_TEAMS
this.transformationsKillswitch = mergedConfig.LOGS_TRANSFORMATIONS_KILLSWITCH
this.retentionEnabledTeamsRaw = mergedConfig.LOGS_RETENTION_ENABLED_TEAMS
@@ -496,11 +500,16 @@ export class LogsIngestionConsumer {
return Promise.resolve(logsSettings.retention_days ?? DEFAULT_LOGS_RETENTION_DAYS)
}
+ private canEmitMetricRules(): boolean {
+ return !this.metricRulesKillswitch && !!this.deps.metricRulesCache && !!this.deps.metricsEmitter
+ }
+
private isMetricRulesEnabledForTeam(teamId: number): boolean {
- if (this.metricRulesKillswitch || !this.deps.metricRulesCache || !this.deps.metricsEmitter) {
- return false
- }
- return teamIdMatchesCsv(this.metricRulesEnabledTeamsRaw, teamId)
+ return this.canEmitMetricRules() && teamIdMatchesCsv(this.metricRulesEnabledTeamsRaw, teamId)
+ }
+
+ private isDefaultMetricRulesEnabledForTeam(teamId: number): boolean {
+ return this.canEmitMetricRules() && teamIdMatchesCsv(this.defaultMetricRulesEnabledTeamsRaw, teamId)
}
// Transformations write hog_function logs, which could come back here as internal logs and loop.
@@ -969,17 +978,24 @@ export class LogsIngestionConsumer {
byTeam: MetricTalliesByTeam,
message: LogsIngestionMessage
): Promise<{ token: string; rules: CompiledMetricRule[]; tallies: BatchTallies } | null> {
- if (!this.appliesTeamLogRules(message) || !this.isMetricRulesEnabledForTeam(message.teamId)) {
+ if (!this.appliesTeamLogRules(message)) {
+ return null
+ }
+ const teamRulesEnabled = this.isMetricRulesEnabledForTeam(message.teamId)
+ const defaultRulesEnabled = this.isDefaultMetricRulesEnabledForTeam(message.teamId)
+ if (!teamRulesEnabled && !defaultRulesEnabled) {
return null
}
let state = byTeam.get(message.teamId)
if (!state) {
- let rules: CompiledMetricRule[]
+ let rules: CompiledMetricRule[] = []
try {
- const all = await this.deps.metricRulesCache!.getCompiledRules(message.teamId)
- // Each consumer tallies only its own record source: the logs consumer runs
- // `logs` rules, the traces consumer runs `spans` rules.
- rules = all.filter((r) => r.source === this.metricRuleSource)
+ if (teamRulesEnabled) {
+ const all = await this.deps.metricRulesCache!.getCompiledRules(message.teamId)
+ // Each consumer tallies only its own record source: the logs consumer runs
+ // `logs` rules, the traces consumer runs `spans` rules.
+ rules = all.filter((r) => r.source === this.metricRuleSource)
+ }
} catch (error) {
// Fail open: metric rules are a purely additive side feature, so a rules-fetch
// failure (e.g. a Postgres blip) must never DLQ or block the log records —
@@ -990,6 +1006,9 @@ export class LogsIngestionConsumer {
})
return null
}
+ if (defaultRulesEnabled) {
+ rules = withDefaultMetricRules(rules, this.metricRuleSource)
+ }
// Re-check after the await: concurrent messages for the same team race here,
// and replacing an existing state would silently drop its accumulated tallies.
state = byTeam.get(message.teamId)
@@ -1085,11 +1104,19 @@ export class LogsIngestionConsumer {
metricRuleState || jsonAttributeKey
? (records: LogRecord[]) => {
if (metricRuleState) {
+ const nowMs = Date.now()
tallyRecords(
metricRuleState.rules,
records,
metricRuleState.tallies,
- Date.now()
+ nowMs,
+ (ruleId, labelKey) =>
+ this.metricSeriesAdmission.admit(
+ message.teamId,
+ ruleId,
+ labelKey,
+ nowMs
+ )
)
}
if (jsonAttributeKey) {
diff --git a/nodejs/src/logs/metrics-rules/default-metric-rules.test.ts b/nodejs/src/logs/metrics-rules/default-metric-rules.test.ts
new file mode 100644
index 000000000000..1308b2a96db3
--- /dev/null
+++ b/nodejs/src/logs/metrics-rules/default-metric-rules.test.ts
@@ -0,0 +1,118 @@
+import type { LogRecord } from '../log-record-avro'
+import { type CompiledMetricRule } from './compile-metric-rules'
+import { DEFAULT_METRIC_RULES, withDefaultMetricRules } from './default-metric-rules'
+import { createBatchTallies, tallyRecords } from './tally'
+
+const NOW_MS = 1_700_000_000_000
+
+const spanRecord = (overrides: Record<string, unknown> = {}): LogRecord =>
+ ({
+ uuid: null,
+ trace_id: null,
+ span_id: null,
+ trace_flags: null,
+ timestamp: NOW_MS * 1000,
+ end_time: NOW_MS * 1000 + 40_000, // 40ms
+ observed_timestamp: null,
+ body: null,
+ severity_text: null,
+ severity_number: null,
+ service_name: 'api',
+ resource_attributes: {},
+ instrumentation_scope: null,
+ event_name: null,
+ attributes: {},
+ name: 'GET /checkout',
+ kind: 2, // SERVER
+ status_code: 1, // OK
+ status_message: null,
+ ...overrides,
+ }) as unknown as LogRecord
+
+const logRecord = (overrides: Partial<LogRecord> = {}): LogRecord => ({
+ uuid: null,
+ trace_id: null,
+ span_id: null,
+ trace_flags: null,
+ timestamp: NOW_MS * 1000,
+ observed_timestamp: null,
+ body: 'hello',
+ severity_text: 'ERROR',
+ severity_number: 17,
+ service_name: 'api',
+ resource_attributes: {},
+ instrumentation_scope: null,
+ event_name: null,
+ attributes: {},
+ ...overrides,
+})
+
+function tallyDefaults(source: 'logs' | 'spans', records: LogRecord[]): Map<string, number> {
+ const rules = DEFAULT_METRIC_RULES[source]
+ const tallies = createBatchTallies()
+ tallyRecords(rules, records, tallies, NOW_MS)
+ const counts = new Map<string, number>()
+ for (const entry of tallies.byRule.get(rules[0]!.id)?.values() ?? []) {
+ counts.set(entry.labelValues.join('|'), entry.count)
+ }
+ return counts
+}
+
+describe('default metric rules', () => {
+ it('names every default metric under the reserved posthog. prefix so it cannot clash with a team rule', () => {
+ for (const rule of [...DEFAULT_METRIC_RULES.spans, ...DEFAULT_METRIC_RULES.logs]) {
+ expect(rule.metricName.startsWith('posthog.')).toBe(true)
+ }
+ })
+
+ it('tallies server, client, producer and consumer spans by service, kind, name and status', () => {
+ const counts = tallyDefaults('spans', [
+ spanRecord(),
+ spanRecord({ status_code: 2 }),
+ spanRecord({ kind: 3, name: 'SELECT orders' }),
+ spanRecord({ kind: 4, name: 'publish events' }),
+ spanRecord({ kind: 5, name: 'consume events' }),
+ ])
+ expect(Object.fromEntries(counts)).toEqual({
+ 'api|SERVER|GET /checkout|OK': 1,
+ 'api|SERVER|GET /checkout|ERROR': 1,
+ 'api|CLIENT|SELECT orders|OK': 1,
+ 'api|PRODUCER|publish events|OK': 1,
+ 'api|CONSUMER|consume events|OK': 1,
+ })
+ })
+
+ it('skips internal spans, whose names are the most numerous and least useful as series', () => {
+ expect(tallyDefaults('spans', [spanRecord({ kind: 1, name: 'render' })]).size).toBe(0)
+ })
+
+ it('counts every log record by service and severity', () => {
+ const counts = tallyDefaults('logs', [
+ logRecord(),
+ logRecord(),
+ logRecord({ severity_text: 'INFO' }),
+ logRecord({ service_name: 'web', severity_text: 'INFO' }),
+ ])
+ expect(Object.fromEntries(counts)).toEqual({ 'api|ERROR': 2, 'api|INFO': 1, 'web|INFO': 1 })
+ })
+
+ describe('withDefaultMetricRules', () => {
+ const userRule: CompiledMetricRule = {
+ id: 'user-rule',
+ metricName: 'checkout.errors',
+ source: 'logs',
+ filterGroup: null,
+ valueAttribute: null,
+ groupBy: [],
+ }
+
+ it('appends the defaults for the source after the team rules', () => {
+ expect(withDefaultMetricRules([userRule], 'logs')).toEqual([userRule, ...DEFAULT_METRIC_RULES.logs])
+ })
+
+ it('lets a team rule with the same metric name win over the default', () => {
+ const shadowing = { ...userRule, metricName: DEFAULT_METRIC_RULES.logs[0]!.metricName }
+ expect(withDefaultMetricRules([shadowing], 'logs')).toEqual([shadowing])
+ })
+ })
+})
diff --git a/nodejs/src/logs/metrics-rules/default-metric-rules.ts b/nodejs/src/logs/metrics-rules/default-metric-rules.ts
new file mode 100644
index 000000000000..f8fb813303fd
--- /dev/null
+++ b/nodejs/src/logs/metrics-rules/default-metric-rules.ts
@@ -0,0 +1,54 @@
+import {
+ type CompiledMetricRule,
+ type MetricRuleRow,
+ type MetricRuleSource,
+ compileMetricRules,
+} from './compile-metric-rules'
+import { SPAN_VALUE_DURATION_MS } from './tally'
+
+const SPAN_KIND_INTERNAL = '1'
+
+/** Built-in rules, not stored rows: the consumer adds them for teams in the default-rules allowlist. */
+const DEFAULT_METRIC_RULE_ROWS: MetricRuleRow[] = [
+ {
+ // INTERNAL spans are the most numerous, and their names are the most likely to carry ids.
+ id: 'default:spans:duration',
+ metric_name: 'posthog.span.duration',
+ source: 'spans',
+ filter_group: {
+ type: 'AND',
+ values: [
+ {
+ type: 'AND',
+ values: [{ key: 'kind', operator: 'is_not', value: SPAN_KIND_INTERNAL, type: 'span_attribute' }],
+ },
+ ],
+ },
+ value_attribute: SPAN_VALUE_DURATION_MS,
+ group_by: ['service_name', 'kind', 'name', 'status_code'],
+ },
+ {
+ id: 'default:logs:records',
+ metric_name: 'posthog.log.records',
+ source: 'logs',
+ filter_group: null,
+ value_attribute: null,
+ group_by: ['service_name', 'severity_text'],
+ },
+]
+
+const compiledDefaults = compileMetricRules(DEFAULT_METRIC_RULE_ROWS)
+
+export const DEFAULT_METRIC_RULES: Record<MetricRuleSource, CompiledMetricRule[]> = {
+ logs: compiledDefaults.filter((rule) => rule.source === 'logs'),
+ spans: compiledDefaults.filter((rule) => rule.source === 'spans'),
+}
+
+/** Appends the source's default rules; a team rule with the same metric name replaces the default. */
+export function withDefaultMetricRules(
+ teamRules: CompiledMetricRule[],
+ source: MetricRuleSource
+): CompiledMetricRule[] {
+ const teamMetricNames = new Set(teamRules.map((rule) => rule.metricName))
+ return [...teamRules, ...DEFAULT_METRIC_RULES[source].filter((rule) => !teamMetricNames.has(rule.metricName))]
+}
diff --git a/nodejs/src/logs/metrics-rules/metric-rules.test.ts b/nodejs/src/logs/metrics-rules/metric-rules.test.ts
index 25108eed3a7d..2f35d6451849 100644
--- a/nodejs/src/logs/metrics-rules/metric-rules.test.ts
+++ b/nodejs/src/logs/metrics-rules/metric-rules.test.ts
@@ -1,6 +1,14 @@
import type { LogRecord } from '../log-record-avro'
import { type MetricRuleRow, compileMetricRules } from './compile-metric-rules'
-import { MAX_LABEL_SETS_PER_RULE, MAX_LABEL_VALUE_LENGTH, createBatchTallies, tallyRecords } from './tally'
+import {
+ DEFAULT_HISTOGRAM_BOUNDS,
+ MAX_LABEL_SETS_PER_RULE,
+ MAX_LABEL_VALUE_LENGTH,
+ MetricSeriesAdmission,
+ OVERFLOW_LABEL_KEY,
+ createBatchTallies,
+ tallyRecords,
+} from './tally'
const NOW_MS = 1_700_000_000_000
@@ -167,17 +175,100 @@ describe('metric rules compile + tally', () => {
expect(entries[0]!.count).toBe(2)
})
- it('caps distinct label sets per rule and counts the overflow', () => {
+ it('folds label sets past the cap into one overflow series instead of dropping them', () => {
const rules = compileMetricRules([ruleRow({ filter_group: null, group_by: ['attributes.request_id'] })])
const tallies = createBatchTallies()
const records = Array.from({ length: MAX_LABEL_SETS_PER_RULE + 3 }, (_, i) =>
record({ attributes: { request_id: `req-${i}` } })
)
tallyRecords(rules, records, tallies, NOW_MS)
- expect(tallies.byRule.get('rule-1')!.size).toBe(MAX_LABEL_SETS_PER_RULE)
+ const ruleTallies = tallies.byRule.get('rule-1')!
+ expect(ruleTallies.size).toBe(MAX_LABEL_SETS_PER_RULE + 1)
+ const overflow = ruleTallies.get(OVERFLOW_LABEL_KEY)!
+ expect(overflow.overflow).toBe(true)
+ expect(overflow.count).toBe(3)
expect(tallies.seriesOverflow.get('rule-1')).toBe(3)
})
+ it('keeps known label sets on their own series after the cap is reached', () => {
+ const rules = compileMetricRules([ruleRow({ filter_group: null, group_by: ['attributes.request_id'] })])
+ const tallies = createBatchTallies()
+ const records = Array.from({ length: MAX_LABEL_SETS_PER_RULE + 1 }, (_, i) =>
+ record({ attributes: { request_id: `req-${i}` } })
+ )
+ records.push(record({ attributes: { request_id: 'req-0' } }))
+ tallyRecords(rules, records, tallies, NOW_MS)
+ const ruleTallies = tallies.byRule.get('rule-1')!
+ expect(ruleTallies.get(JSON.stringify(['req-0']))!.count).toBe(2)
+ expect(ruleTallies.get(OVERFLOW_LABEL_KEY)!.count).toBe(1)
+ })
+
+ it('bounds label sets per rule across batches until the hour changes', () => {
+ const rules = compileMetricRules([ruleRow({ filter_group: null, group_by: ['attributes.request_id'] })])
+ const admission = new MetricSeriesAdmission()
+ const tallyBatch = (ids: string[], nowMs: number): Map<string, { count: number }> => {
+ const tallies = createBatchTallies()
+ const records = ids.map((id) => record({ attributes: { request_id: id }, timestamp: nowMs * 1000 }))
+ tallyRecords(rules, records, tallies, nowMs, (ruleId, labelKey) =>
+ admission.admit(1, ruleId, labelKey, nowMs)
+ )
+ return tallies.byRule.get('rule-1')!
+ }
+ tallyBatch(
+ Array.from({ length: MAX_LABEL_SETS_PER_RULE }, (_, i) => `req-${i}`),
+ NOW_MS
+ )
+
+ const sameHour = tallyBatch(['req-0', 'req-new'], NOW_MS + 1000)
+ expect(sameHour.get(JSON.stringify(['req-0']))!.count).toBe(1)
+ expect(sameHour.get(JSON.stringify(['req-new']))).toBeUndefined()
+ expect(sameHour.get(OVERFLOW_LABEL_KEY)!.count).toBe(1)
+
+ const nextHour = tallyBatch(['req-new'], NOW_MS + 60 * 60 * 1000)
+ expect(nextHour.get(JSON.stringify(['req-new']))!.count).toBe(1)
+ })
+
+ it('counts colliding label sets separately, so crafted FNV-1a collisions cannot pass the cap', () => {
+ const admission = new MetricSeriesAdmission(1)
+ expect(admission.admit(1, 'rule-a', 'costarring', NOW_MS)).toBe(true)
+ expect(admission.admit(1, 'rule-a', 'liquid', NOW_MS)).toBe(false)
+ })
+
+ it('caps a team across all its rules, so rule churn cannot take the budget of other teams', () => {
+ const admission = new MetricSeriesAdmission(2, 3, 10)
+ expect(admission.admit(1, 'rule-a', 'x', NOW_MS)).toBe(true)
+ expect(admission.admit(1, 'rule-a', 'y', NOW_MS)).toBe(true)
+ expect(admission.admit(1, 'rule-b', 'x', NOW_MS)).toBe(true)
+ expect(admission.admit(1, 'rule-c', 'x', NOW_MS)).toBe(false)
+ expect(admission.admit(1, 'rule-a', 'x', NOW_MS)).toBe(true)
+ expect(admission.admit(2, 'rule-a', 'x', NOW_MS)).toBe(true)
+ })
+
+ it('stops admitting new label sets when the process-wide budget is spent', () => {
+ const admission = new MetricSeriesAdmission(2, 2, 3)
+ expect(admission.admit(1, 'rule-a', 'x', NOW_MS)).toBe(true)
+ expect(admission.admit(1, 'rule-a', 'y', NOW_MS)).toBe(true)
+ expect(admission.admit(2, 'rule-a', 'x', NOW_MS)).toBe(true)
+ expect(admission.admit(2, 'rule-b', 'z', NOW_MS)).toBe(false)
+ expect(admission.admit(2, 'rule-b', 'z', NOW_MS + 60 * 60 * 1000)).toBe(true)
+ })
+
+ it.each([
+ [0, 0],
+ [5, 1],
+ [5.01, 2],
+ [10_000, DEFAULT_HISTOGRAM_BOUNDS.length - 1],
+ [10_001, DEFAULT_HISTOGRAM_BOUNDS.length],
+ ])('puts value %p in histogram bucket %p (upper bound inclusive)', (value, bucket) => {
+ const rules = compileMetricRules([ruleRow({ filter_group: null, value_attribute: 'attributes.v' })])
+ const tallies = createBatchTallies()
+ tallyRecords(rules, [record({ attributes: { v: String(value) } })], tallies, NOW_MS)
+ const entry = [...tallies.byRule.get('rule-1')!.values()][0]!
+ const expected = new Array(DEFAULT_HISTOGRAM_BOUNDS.length + 1).fill(0)
+ expected[bucket] = 1
+ expect(entry.bucketCounts).toEqual(expected)
+ })
+
it('captures an exemplar trace id from a matching record and ignores zeroed trace ids', () => {
const rules = compileMetricRules([ruleRow({ filter_group: null })])
const tallies = createBatchTallies()
diff --git a/nodejs/src/logs/metrics-rules/otlp-payload.test.ts b/nodejs/src/logs/metrics-rules/otlp-payload.test.ts
index dc26419b4964..d57d25f0efa1 100644
--- a/nodejs/src/logs/metrics-rules/otlp-payload.test.ts
+++ b/nodejs/src/logs/metrics-rules/otlp-payload.test.ts
@@ -1,7 +1,7 @@
import type { LogRecord } from '../log-record-avro'
import { compileMetricRules } from './compile-metric-rules'
import { buildMetricRulesOtlpPayload } from './otlp-payload'
-import { createBatchTallies, tallyRecords } from './tally'
+import { DEFAULT_HISTOGRAM_BOUNDS, MAX_LABEL_SETS_PER_RULE, createBatchTallies, tallyRecords } from './tally'
const NOW_MS = 1_700_000_000_123
// Snapped down to the enclosing 10s bucket, in nanos.
@@ -69,7 +69,7 @@ describe('buildMetricRulesOtlpPayload', () => {
expect(errorPoint.timeUnixNano).toBe(SNAPPED_NANOS)
})
- it('emits a value-attribute rule as a delta histogram carrying count and sum', () => {
+ it('emits a value-attribute rule as a delta histogram with explicit bucket counts', () => {
const payload = build([distributionRule], [record(), record({ attributes: { duration_ms: '7.5' } })])
const metric = payload.resourceMetrics[0].scopeMetrics[0].metrics[0]
expect(metric.name).toBe('log.request_duration')
@@ -77,9 +77,31 @@ describe('buildMetricRulesOtlpPayload', () => {
const dp = metric.histogram.dataPoints[0]
expect(dp.count).toBe(2)
expect(dp.sum).toBe(20)
- expect(dp.explicitBounds).toEqual([])
- expect(dp.bucketCounts).toEqual([2])
+ expect(dp.explicitBounds).toEqual(DEFAULT_HISTOGRAM_BOUNDS)
+ const expectedCounts = new Array(DEFAULT_HISTOGRAM_BOUNDS.length + 1).fill(0)
+ expectedCounts[2] = 1 // 7.5 falls in (5, 10]
+ expectedCounts[3] = 1 // 12.5 falls in (10, 25]
+ expect(dp.bucketCounts).toEqual(expectedCounts)
+ expect(dp.attributes).toEqual([{ key: 'posthog.bucket_layout', value: { stringValue: 'otel_default_v1' } }])
expect(dp.timeUnixNano).toBe(SNAPPED_NANOS)
+ expect(metric.unit).toBeUndefined()
+ })
+
+ it('emits a span duration rule with the ms unit', () => {
+ const span = { ...record(), end_time: NOW_MS * 1000 + 40_000 } as LogRecord
+ const payload = build([{ ...distributionRule, source: 'spans', value_attribute: 'duration_ms' }], [span])
+ expect(payload.resourceMetrics[0].scopeMetrics[0].metrics[0].unit).toBe('ms')
+ })
+
+ it('emits the overflow series with only the otel.metric.overflow attribute and no service', () => {
+ const records = Array.from({ length: MAX_LABEL_SETS_PER_RULE + 2 }, (_, i) =>
+ record({ service_name: `svc-${i}` })
+ )
+ const payload = build([{ ...countRule, group_by: ['service_name', 'severity_text'] }], records)
+ const overflowResource = payload.resourceMetrics.find((rm: any) => rm.resource.attributes.length === 0)
+ const dp = overflowResource.scopeMetrics[0].metrics[0].sum.dataPoints[0]
+ expect(dp.attributes).toEqual([{ key: 'otel.metric.overflow', value: { boolValue: true } }])
+ expect(dp.asDouble).toBe(2)
})
it('strips the attributes. prefix from group-by label names', () => {
diff --git a/nodejs/src/logs/metrics-rules/otlp-payload.ts b/nodejs/src/logs/metrics-rules/otlp-payload.ts
index 4858def399a3..73a26f525197 100644
--- a/nodejs/src/logs/metrics-rules/otlp-payload.ts
+++ b/nodejs/src/logs/metrics-rules/otlp-payload.ts
@@ -1,5 +1,5 @@
import type { CompiledMetricRule } from './compile-metric-rules'
-import type { BatchTallies, MetricTallyEntry } from './tally'
+import { type BatchTallies, DEFAULT_HISTOGRAM_BOUNDS, type MetricTallyEntry, SPAN_VALUE_DURATION_MS } from './tally'
/**
* Hand-built OTLP/JSON `ExportMetricsServiceRequest` for log-generated metrics.
@@ -20,6 +20,15 @@ const OTLP_TEMPORALITY_DELTA = 1
const ATTRIBUTE_PREFIXES = ['attributes.', 'resource_attributes.']
+/** OTel SDK convention for the series that collects data points past a cardinality limit. */
+const OVERFLOW_ATTRIBUTE = { key: 'otel.metric.overflow', value: { boolValue: true } }
+
+/**
+ * Series identity does not include bucket bounds, and samples keep one bounds array per series-hour.
+ * This label gives each bucket layout its own series, so samples with different layouts never merge.
+ */
+const BUCKET_LAYOUT_ATTRIBUTE = { key: 'posthog.bucket_layout', value: { stringValue: 'otel_default_v1' } }
+
/** Label name as it appears on the emitted series: map keys lose their prefix (Datadog drops `@` the same way). */
function labelName(groupByKey: string): string {
for (const prefix of ATTRIBUTE_PREFIXES) {
@@ -53,7 +62,7 @@ export function buildMetricRulesOtlpPayload(
}
const serviceIndex = rule.groupBy.indexOf('service_name')
for (const entry of ruleTallies.values()) {
- const service = serviceIndex >= 0 ? (entry.labelValues[serviceIndex] ?? '') : ''
+ const service = serviceIndex >= 0 && !entry.overflow ? (entry.labelValues[serviceIndex] ?? '') : ''
let ruleMap = byService.get(service)
if (!ruleMap) {
ruleMap = new Map()
@@ -101,15 +110,14 @@ function serializeRuleMetric(
if (rule.valueAttribute) {
return {
name: rule.metricName,
+ ...(rule.valueAttribute === SPAN_VALUE_DURATION_MS ? { unit: 'ms' } : {}),
histogram: {
dataPoints: entries.map((entry) => ({
- ...dataPointBase(rule, entry, startTimeUnixNano, timeUnixNano),
+ ...dataPointBase(rule, entry, startTimeUnixNano, timeUnixNano, BUCKET_LAYOUT_ATTRIBUTE),
count: entry.count,
sum: entry.sum,
- // No percentile buckets yet: a single catch-all bucket keeps the shape a
- // valid OTLP histogram while carrying only count + sum.
- bucketCounts: [entry.count],
- explicitBounds: [],
+ bucketCounts: entry.bucketCounts,
+ explicitBounds: DEFAULT_HISTOGRAM_BOUNDS,
})),
aggregationTemporality: OTLP_TEMPORALITY_DELTA,
},
@@ -132,10 +140,14 @@ function dataPointBase(
rule: CompiledMetricRule,
entry: MetricTallyEntry,
startTimeUnixNano: string,
- timeUnixNano: string
+ timeUnixNano: string,
+ extraAttribute?: Record<string, unknown>
): Record<string, unknown> {
- const attributes: Record<string, unknown>[] = []
- for (let i = 0; i < rule.groupBy.length; i++) {
+ const attributes: Record<string, unknown>[] = entry.overflow ? [OVERFLOW_ATTRIBUTE] : []
+ if (extraAttribute) {
+ attributes.push(extraAttribute)
+ }
+ for (let i = 0; i < rule.groupBy.length && !entry.overflow; i++) {
const key = rule.groupBy[i]!
if (key === 'service_name') {
continue // carried on the resource block instead
diff --git a/nodejs/src/logs/metrics-rules/tally.ts b/nodejs/src/logs/metrics-rules/tally.ts
index 6e0fa34db1ca..962ea524d9eb 100644
--- a/nodejs/src/logs/metrics-rules/tally.ts
+++ b/nodejs/src/logs/metrics-rules/tally.ts
@@ -1,3 +1,5 @@
+import { createHmac, randomBytes } from 'crypto'
+
import { decodeLogAttributeValue } from '../attribute-value'
import type { LogRecord } from '../log-record-avro'
import { matchFilterGroup } from '../sampling/filter-group-match'
@@ -6,6 +8,14 @@ import type { CompiledMetricRule } from './compile-metric-rules'
/** Cap on distinct group-by label sets per rule per batch — bounds emitted series cardinality. */
export const MAX_LABEL_SETS_PER_RULE = 1000
+/** Not valid JSON, so no real label set (always a JSON array) can collide with it. */
+export const OVERFLOW_LABEL_KEY = '__overflow__'
+
+/** The OTel SDK default explicit bucket bounds, so percentiles match an SDK-sent histogram. */
+export const DEFAULT_HISTOGRAM_BOUNDS: readonly number[] = [
+ 0, 5, 10, 25, 50, 75, 100, 250, 500, 750, 1000, 2500, 5000, 7500, 10000,
+]
+
export const MAX_LABEL_VALUE_LENGTH = 256
/**
@@ -34,13 +44,15 @@ export type MetricTallyEntry = {
labelValues: string[]
count: number
sum: number
+ bucketCounts: number[] | null
+ overflow: boolean
exemplarTraceId: string | null
exemplarSpanId: string | null
}
export type BatchTallies = {
byRule: Map<string, Map<string, MetricTallyEntry>>
- /** Records that matched but were dropped because the rule hit MAX_LABEL_SETS_PER_RULE. */
+ /** Records that matched after the rule hit its label-set cap, tallied into the overflow series. */
seriesOverflow: Map<string, number>
/** Records that matched a value-attribute rule but carried a missing/non-numeric value. */
valueSkipped: number
@@ -114,6 +126,93 @@ function resolveLabelValue(key: string, record: LogRecord): string {
return value.length > MAX_LABEL_VALUE_LENGTH ? value.slice(0, MAX_LABEL_VALUE_LENGTH) : value
}
+const SERIES_WINDOW_MS = 60 * 60 * 1000
+
+export const MAX_ADMITTED_LABEL_SETS_PER_TEAM = 5_000
+export const MAX_ADMITTED_LABEL_SETS = 500_000
+
+/**
+ * HMAC with a random per-process key, truncated to 48 bits. A team cannot craft colliding labels
+ * offline, so a collision cannot admit extra label sets past the cap.
+ */
+const LABEL_HASH_KEY = randomBytes(32)
+
+function hashLabelKey(labelKey: string): number {
+ return createHmac('sha256', LABEL_HASH_KEY).update(labelKey).digest().readUIntBE(0, 6)
+}
+
+/**
+ * Bounds the label sets each rule admits per hour in this process. Batch tallies start empty,
+ * so without this a rule whose labels change every batch would store new series without limit.
+ * The window matches the hourly series grain of metric samples.
+ */
+export class MetricSeriesAdmission {
+ private window = -1
+ private admitted = new Map<string, Set<number>>()
+ // Counts survive rule deletion until the hour changes, so rule churn cannot reset a team's budget.
+ private admittedByTeam = new Map<number, number>()
+ private admittedTotal = 0
+
+ constructor(
+ private readonly maxPerRule: number = MAX_LABEL_SETS_PER_RULE,
+ private readonly maxPerTeam: number = MAX_ADMITTED_LABEL_SETS_PER_TEAM,
+ private readonly maxTotal: number = MAX_ADMITTED_LABEL_SETS
+ ) {}
+
+ public admit(teamId: number, ruleId: string, labelKey: string, nowMs: number): boolean {
+ const window = Math.floor(nowMs / SERIES_WINDOW_MS)
+ if (window !== this.window) {
+ this.admitted.clear()
+ this.admittedByTeam.clear()
+ this.admittedTotal = 0
+ this.window = window
+ }
+ const key = `${teamId}:${ruleId}`
+ const labelHash = hashLabelKey(labelKey)
+ let labelHashes = this.admitted.get(key)
+ if (labelHashes?.has(labelHash)) {
+ return true
+ }
+ const teamAdmitted = this.admittedByTeam.get(teamId) ?? 0
+ if (
+ (labelHashes?.size ?? 0) >= this.maxPerRule ||
+ teamAdmitted >= this.maxPerTeam ||
+ this.admittedTotal >= this.maxTotal
+ ) {
+ return false
+ }
+ if (!labelHashes) {
+ labelHashes = new Set()
+ this.admitted.set(key, labelHashes)
+ }
+ labelHashes.add(labelHash)
+ this.admittedByTeam.set(teamId, teamAdmitted + 1)
+ this.admittedTotal++
+ return true
+ }
+}
+
+function histogramBucketIndex(value: number): number {
+ for (let i = 0; i < DEFAULT_HISTOGRAM_BOUNDS.length; i++) {
+ if (value <= DEFAULT_HISTOGRAM_BOUNDS[i]!) {
+ return i
+ }
+ }
+ return DEFAULT_HISTOGRAM_BOUNDS.length
+}
+
+function newTallyEntry(labelValues: string[], withBuckets: boolean, overflow: boolean): MetricTallyEntry {
+ return {
+ labelValues,
+ count: 0,
+ sum: 0,
+ bucketCounts: withBuckets ? new Array<number>(DEFAULT_HISTOGRAM_BOUNDS.length + 1).fill(0) : null,
+ overflow,
+ exemplarTraceId: null,
+ exemplarSpanId: null,
+ }
+}
+
function resolveNumericValue(key: string, record: LogRecord): number | null {
// Span wall-clock duration from Avro timestamp-micros, no attribute lookup.
if (key === SPAN_VALUE_DURATION_MS) {
@@ -166,7 +265,8 @@ export function tallyRecords(
rules: CompiledMetricRule[],
records: LogRecord[],
tallies: BatchTallies,
- nowMs: number
+ nowMs: number,
+ admitLabelSet?: (ruleId: string, labelKey: string) => boolean
): void {
if (rules.length === 0 || records.length === 0) {
return
@@ -207,7 +307,7 @@ export function tallyRecords(
}
const labelValues = rule.groupBy.map((key) => resolveLabelValue(key, record))
- const labelKey = JSON.stringify(labelValues)
+ let labelKey = JSON.stringify(labelValues)
let ruleTallies = tallies.byRule.get(rule.id)
if (!ruleTallies) {
@@ -216,15 +316,32 @@ export function tallyRecords(
}
let entry = ruleTallies.get(labelKey)
if (!entry) {
- if (ruleTallies.size >= MAX_LABEL_SETS_PER_RULE) {
+ const overflowEntry = ruleTallies.get(OVERFLOW_LABEL_KEY)
+ const realLabelSets = ruleTallies.size - (overflowEntry ? 1 : 0)
+ const admitted = realLabelSets < MAX_LABEL_SETS_PER_RULE && (admitLabelSet?.(rule.id, labelKey) ?? true)
+ if (!admitted) {
+ // Past the cap the record still counts, on one shared series, so rule totals stay correct.
tallies.seriesOverflow.set(rule.id, (tallies.seriesOverflow.get(rule.id) ?? 0) + 1)
- continue
+ labelKey = OVERFLOW_LABEL_KEY
+ entry = overflowEntry
+ if (!entry) {
+ entry = newTallyEntry(
+ rule.groupBy.map(() => ''),
+ rule.valueAttribute !== null,
+ true
+ )
+ ruleTallies.set(labelKey, entry)
+ }
+ } else {
+ entry = newTallyEntry(labelValues, rule.valueAttribute !== null, false)
+ ruleTallies.set(labelKey, entry)
}
- entry = { labelValues, count: 0, sum: 0, exemplarTraceId: null, exemplarSpanId: null }
- ruleTallies.set(labelKey, entry)
}
entry.count += 1
entry.sum += value
+ if (entry.bucketCounts) {
+ entry.bucketCounts[histogramBucketIndex(value)]! += 1
+ }
if (!entry.exemplarTraceId) {
const traceIdHex = idToHex(record.trace_id, 16)
if (traceIdHex) {
diff --git a/nodejs/src/logs/traces-ingestion-consumer.ts b/nodejs/src/logs/traces-ingestion-consumer.ts
index cae16c23e4de..437e58f57591 100644
--- a/nodejs/src/logs/traces-ingestion-consumer.ts
+++ b/nodejs/src/logs/traces-ingestion-consumer.ts
@@ -50,6 +50,7 @@ export class TracesIngestionConsumer extends LogsIngestionConsumer {
LOGS_METRICS_RULES_ENABLED_TEAMS: config.TRACES_METRICS_RULES_ENABLED_TEAMS,
LOGS_METRICS_RULES_KILLSWITCH: config.TRACES_METRICS_RULES_KILLSWITCH,
LOGS_METRICS_RULES_EXPORT_URL: config.TRACES_METRICS_RULES_EXPORT_URL,
+ LOGS_DEFAULT_METRIC_RULES_ENABLED_TEAMS: config.TRACES_DEFAULT_METRIC_RULES_ENABLED_TEAMS,
// Span retention gates on traces-specific env config so it rolls out
// independently of log retention.
LOGS_RETENTION_ENABLED_TEAMS: config.TRACES_RETENTION_ENABLED_TEAMS,