:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
chore(logs): share the test avro fields and return metric tallies to the caller (#122027)
Jon McCallum14482616db79
diff --git a/nodejs/src/logs/logs-ingestion-consumer.test.ts b/nodejs/src/logs/logs-ingestion-consumer.test.tsindex abc939516536..b777f96f44b1 100644--- a/nodejs/src/logs/logs-ingestion-consumer.test.ts+++ b/nodejs/src/logs/logs-ingestion-consumer.test.ts@@ -75,6 +75,23 @@ jest.mock('~/common/utils/posthog', () => {let offsetIncrementer = 0+const LOG_RECORD_BASE_FIELDS = [+ { name: 'uuid', type: ['null', 'string'] },+ { name: 'trace_id', type: ['null', 'bytes'] },+ { name: 'span_id', type: ['null', 'bytes'] },+ { name: 'trace_flags', type: ['null', 'int'] },+ { name: 'timestamp', type: ['null', 'long'] },+ { name: 'observed_timestamp', type: ['null', 'long'] },+ { name: 'body', type: ['null', 'string'] },+ { name: 'severity_text', type: ['null', 'string'] },+ { name: 'severity_number', type: ['null', 'int'] },+ { name: 'service_name', type: ['null', 'string'] },+ { name: 'resource_attributes', type: ['null', { type: 'map', values: 'string' }] },+ { name: 'instrumentation_scope', type: ['null', 'string'] },+ { name: 'event_name', type: ['null', 'string'] },+ { name: 'attributes', type: ['null', { type: 'map', values: 'string' }] },+]+const createKafkaMessage = async (logData: any,headers: Record<string, string> = {},@@ -104,20 +121,7 @@ const createKafkaMessage = async (type: 'record',name: 'LogRecord',fields: [- { name: 'uuid', type: ['null', 'string'] },- { name: 'trace_id', type: ['null', 'bytes'] },- { name: 'span_id', type: ['null', 'bytes'] },- { name: 'trace_flags', type: ['null', 'int'] },- { name: 'timestamp', type: ['null', 'long'] },- { name: 'observed_timestamp', type: ['null', 'long'] },- { name: 'body', type: ['null', 'string'] },- { name: 'severity_text', type: ['null', 'string'] },- { name: 'severity_number', type: ['null', 'int'] },- { name: 'service_name', type: ['null', 'string'] },- { name: 'resource_attributes', type: ['null', { type: 'map', values: 'string' }] },- { name: 'instrumentation_scope', type: ['null', 'string'] },- { name: 'event_name', type: ['null', 'string'] },- { name: 'attributes', type: ['null', { type: 'map', values: 'string' }] },+ ...LOG_RECORD_BASE_FIELDS,{ name: 'pattern', type: ['null', 'string'], default: null },{ name: 'pattern_version', type: ['null', 'int'], default: null },],@@ -153,22 +157,7 @@ const createMultiRecordKafkaMessage = async (const logRecordType = avro.Type.forSchema({type: 'record',name: 'LogRecord',- fields: [- { name: 'uuid', type: ['null', 'string'] },- { name: 'trace_id', type: ['null', 'bytes'] },- { name: 'span_id', type: ['null', 'bytes'] },- { name: 'trace_flags', type: ['null', 'int'] },- { name: 'timestamp', type: ['null', 'long'] },- { name: 'observed_timestamp', type: ['null', 'long'] },- { name: 'body', type: ['null', 'string'] },- { name: 'severity_text', type: ['null', 'string'] },- { name: 'severity_number', type: ['null', 'int'] },- { name: 'service_name', type: ['null', 'string'] },- { name: 'resource_attributes', type: ['null', { type: 'map', values: 'string' }] },- { name: 'instrumentation_scope', type: ['null', 'string'] },- { name: 'event_name', type: ['null', 'string'] },- { name: 'attributes', type: ['null', { type: 'map', values: 'string' }] },- ],+ fields: LOG_RECORD_BASE_FIELDS,})const records: LogRecord[] = logDataList.map((logData, i) => ({diff --git a/nodejs/src/logs/logs-ingestion-consumer.ts b/nodejs/src/logs/logs-ingestion-consumer.tsindex 7c7613f6c7f1..1747b1d74aae 100644--- a/nodejs/src/logs/logs-ingestion-consumer.ts+++ b/nodejs/src/logs/logs-ingestion-consumer.ts@@ -355,6 +355,8 @@ function makeTransformStage(transform: LogRecordsTransform): PipelineStage {}}+type MetricTalliesByTeam = Map<number, { token: string; rules: CompiledMetricRule[]; tallies: BatchTallies }>+export class LogsIngestionConsumer {protected name = 'LogsIngestionConsumer'// Billing identity for quota enforcement and usage metering; overridden by subclasses (e.g. traces).@@ -700,11 +702,12 @@ export class LogsIngestionConsumer {return {// Produce first so PII replacement counts are folded into `usageStats` before MSK usage emitbackgroundTask: (async () => {- await this.processAndProduceLogMessages(+ const metricTalliesByTeam = await this.processAndProduceLogMessages(rateLimiterAllowedMessages,usageStats,transformationBatchBudget)+ await this.emitMetricRuleTallies(metricTalliesByTeam)await this.deps.teamManager.setTeamsIngestedData(rateLimiterAllowedMessages.map((message) => message.teamId))@@ -888,7 +891,7 @@ export class LogsIngestionConsumer {* as one OTLP payload per team after produce.*/private async getMetricRuleBatchState(- byTeam: Map<number, { token: string; rules: CompiledMetricRule[]; tallies: BatchTallies }>,+ byTeam: MetricTalliesByTeam,message: LogsIngestionMessage): Promise<{ token: string; rules: CompiledMetricRule[]; tallies: BatchTallies } | null> {if (!this.isMetricRulesEnabledForTeam(message.teamId)) {@@ -923,9 +926,7 @@ export class LogsIngestionConsumer {return state.rules.length > 0 ? state : null}- private async emitMetricRuleTallies(- byTeam: Map<number, { token: string; rules: CompiledMetricRule[]; tallies: BatchTallies }>- ): Promise<void> {+ private async emitMetricRuleTallies(byTeam: MetricTalliesByTeam): Promise<void> {for (const [teamId, { token, rules, tallies }] of byTeam) {const teamIdLabel = teamId.toString()if (tallies.valueSkipped > 0) {@@ -968,11 +969,8 @@ export class LogsIngestionConsumer {messages: LogsIngestionMessage[],usageStats: UsageStatsByTeam,transformationBatchBudget?: TransformationBatchBudget- ): Promise<void> {- const metricTalliesByTeam = new Map<- number,- { token: string; rules: CompiledMetricRule[]; tallies: BatchTallies }- >()+ ): Promise<MetricTalliesByTeam> {+ const metricTalliesByTeam: MetricTalliesByTeam = new Map()const limit = pLimit(MAX_CONCURRENT_MESSAGE_PROCESSES)let quarantined = 0const results = await Promise.allSettled(@@ -1209,7 +1207,7 @@ export class LogsIngestionConsumer {throw failures[0].reason}- await this.emitMetricRuleTallies(metricTalliesByTeam)+ return metricTalliesByTeam}private async produceToDlq(message: LogsIngestionMessage, error: unknown): Promise<void> {