Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

chore(logs): share the test avro fields and return metric tallies to the caller (#122027)

Jon McCallum14482616db79
diff --git a/nodejs/src/logs/logs-ingestion-consumer.test.ts b/nodejs/src/logs/logs-ingestion-consumer.test.ts
index abc939516536..b777f96f44b1 100644
--- a/nodejs/src/logs/logs-ingestion-consumer.test.ts
+++ b/nodejs/src/logs/logs-ingestion-consumer.test.ts
@@ -75,6 +75,23 @@ jest.mock('~/common/utils/posthog', () => {
let offsetIncrementer = 0
+const LOG_RECORD_BASE_FIELDS = [
+ { name: 'uuid', type: ['null', 'string'] },
+ { name: 'trace_id', type: ['null', 'bytes'] },
+ { name: 'span_id', type: ['null', 'bytes'] },
+ { name: 'trace_flags', type: ['null', 'int'] },
+ { name: 'timestamp', type: ['null', 'long'] },
+ { name: 'observed_timestamp', type: ['null', 'long'] },
+ { name: 'body', type: ['null', 'string'] },
+ { name: 'severity_text', type: ['null', 'string'] },
+ { name: 'severity_number', type: ['null', 'int'] },
+ { name: 'service_name', type: ['null', 'string'] },
+ { name: 'resource_attributes', type: ['null', { type: 'map', values: 'string' }] },
+ { name: 'instrumentation_scope', type: ['null', 'string'] },
+ { name: 'event_name', type: ['null', 'string'] },
+ { name: 'attributes', type: ['null', { type: 'map', values: 'string' }] },
+]
+
const createKafkaMessage = async (
logData: any,
headers: Record<string, string> = {},
@@ -104,20 +121,7 @@ const createKafkaMessage = async (
type: 'record',
name: 'LogRecord',
fields: [
- { name: 'uuid', type: ['null', 'string'] },
- { name: 'trace_id', type: ['null', 'bytes'] },
- { name: 'span_id', type: ['null', 'bytes'] },
- { name: 'trace_flags', type: ['null', 'int'] },
- { name: 'timestamp', type: ['null', 'long'] },
- { name: 'observed_timestamp', type: ['null', 'long'] },
- { name: 'body', type: ['null', 'string'] },
- { name: 'severity_text', type: ['null', 'string'] },
- { name: 'severity_number', type: ['null', 'int'] },
- { name: 'service_name', type: ['null', 'string'] },
- { name: 'resource_attributes', type: ['null', { type: 'map', values: 'string' }] },
- { name: 'instrumentation_scope', type: ['null', 'string'] },
- { name: 'event_name', type: ['null', 'string'] },
- { name: 'attributes', type: ['null', { type: 'map', values: 'string' }] },
+ ...LOG_RECORD_BASE_FIELDS,
{ name: 'pattern', type: ['null', 'string'], default: null },
{ name: 'pattern_version', type: ['null', 'int'], default: null },
],
@@ -153,22 +157,7 @@ const createMultiRecordKafkaMessage = async (
const logRecordType = avro.Type.forSchema({
type: 'record',
name: 'LogRecord',
- fields: [
- { name: 'uuid', type: ['null', 'string'] },
- { name: 'trace_id', type: ['null', 'bytes'] },
- { name: 'span_id', type: ['null', 'bytes'] },
- { name: 'trace_flags', type: ['null', 'int'] },
- { name: 'timestamp', type: ['null', 'long'] },
- { name: 'observed_timestamp', type: ['null', 'long'] },
- { name: 'body', type: ['null', 'string'] },
- { name: 'severity_text', type: ['null', 'string'] },
- { name: 'severity_number', type: ['null', 'int'] },
- { name: 'service_name', type: ['null', 'string'] },
- { name: 'resource_attributes', type: ['null', { type: 'map', values: 'string' }] },
- { name: 'instrumentation_scope', type: ['null', 'string'] },
- { name: 'event_name', type: ['null', 'string'] },
- { name: 'attributes', type: ['null', { type: 'map', values: 'string' }] },
- ],
+ fields: LOG_RECORD_BASE_FIELDS,
})
const records: LogRecord[] = logDataList.map((logData, i) => ({
diff --git a/nodejs/src/logs/logs-ingestion-consumer.ts b/nodejs/src/logs/logs-ingestion-consumer.ts
index 7c7613f6c7f1..1747b1d74aae 100644
--- a/nodejs/src/logs/logs-ingestion-consumer.ts
+++ b/nodejs/src/logs/logs-ingestion-consumer.ts
@@ -355,6 +355,8 @@ function makeTransformStage(transform: LogRecordsTransform): PipelineStage {
}
}
+type MetricTalliesByTeam = Map<number, { token: string; rules: CompiledMetricRule[]; tallies: BatchTallies }>
+
export class LogsIngestionConsumer {
protected name = 'LogsIngestionConsumer'
// Billing identity for quota enforcement and usage metering; overridden by subclasses (e.g. traces).
@@ -700,11 +702,12 @@ export class LogsIngestionConsumer {
return {
// Produce first so PII replacement counts are folded into `usageStats` before MSK usage emit
backgroundTask: (async () => {
- await this.processAndProduceLogMessages(
+ const metricTalliesByTeam = await this.processAndProduceLogMessages(
rateLimiterAllowedMessages,
usageStats,
transformationBatchBudget
)
+ await this.emitMetricRuleTallies(metricTalliesByTeam)
await this.deps.teamManager.setTeamsIngestedData(
rateLimiterAllowedMessages.map((message) => message.teamId)
)
@@ -888,7 +891,7 @@ export class LogsIngestionConsumer {
* as one OTLP payload per team after produce.
*/
private async getMetricRuleBatchState(
- byTeam: Map<number, { token: string; rules: CompiledMetricRule[]; tallies: BatchTallies }>,
+ byTeam: MetricTalliesByTeam,
message: LogsIngestionMessage
): Promise<{ token: string; rules: CompiledMetricRule[]; tallies: BatchTallies } | null> {
if (!this.isMetricRulesEnabledForTeam(message.teamId)) {
@@ -923,9 +926,7 @@ export class LogsIngestionConsumer {
return state.rules.length > 0 ? state : null
}
- private async emitMetricRuleTallies(
- byTeam: Map<number, { token: string; rules: CompiledMetricRule[]; tallies: BatchTallies }>
- ): Promise<void> {
+ private async emitMetricRuleTallies(byTeam: MetricTalliesByTeam): Promise<void> {
for (const [teamId, { token, rules, tallies }] of byTeam) {
const teamIdLabel = teamId.toString()
if (tallies.valueSkipped > 0) {
@@ -968,11 +969,8 @@ export class LogsIngestionConsumer {
messages: LogsIngestionMessage[],
usageStats: UsageStatsByTeam,
transformationBatchBudget?: TransformationBatchBudget
- ): Promise<void> {
- const metricTalliesByTeam = new Map<
- number,
- { token: string; rules: CompiledMetricRule[]; tallies: BatchTallies }
- >()
+ ): Promise<MetricTalliesByTeam> {
+ const metricTalliesByTeam: MetricTalliesByTeam = new Map()
const limit = pLimit(MAX_CONCURRENT_MESSAGE_PROCESSES)
let quarantined = 0
const results = await Promise.allSettled(
@@ -1209,7 +1207,7 @@ export class LogsIngestionConsumer {
throw failures[0].reason
}
- await this.emitMetricRuleTallies(metricTalliesByTeam)
+ return metricTalliesByTeam
}
private async produceToDlq(message: LogsIngestionMessage, error: unknown): Promise<void> {