Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

chore(ingestion): remove legacy metrics ingestion consumer (#128375)

Frank Hamand2f59876c951b
diff --git a/nodejs/src/ingestion/pipelines/metrics/config.ts b/nodejs/src/ingestion/pipelines/metrics/config.ts
index 7fc7d098b800..9d38b663f577 100644
--- a/nodejs/src/ingestion/pipelines/metrics/config.ts
+++ b/nodejs/src/ingestion/pipelines/metrics/config.ts
@@ -29,7 +29,6 @@ export type MetricsIngestionConsumerConfig = {
METRICS_INGESTION_CONSUMER_CONSUME_TOPIC: string
METRICS_INGESTION_CONSUMER_DLQ_TOPIC: string
METRICS_INGESTION_CONSUMER_CLICKHOUSE_TOPIC: string
- METRICS_INGESTION_USE_PIPELINE_FRAMEWORK: boolean
METRICS_REDIS_HOST: string
METRICS_REDIS_PORT: number
METRICS_REDIS_PASSWORD: string
@@ -53,7 +52,6 @@ export function getDefaultMetricsIngestionConsumerConfig(): MetricsIngestionCons
METRICS_INGESTION_CONSUMER_CONSUME_TOPIC: KAFKA_METRICS_INGESTION,
METRICS_INGESTION_CONSUMER_DLQ_TOPIC: KAFKA_METRICS_INGESTION_DLQ,
METRICS_INGESTION_CONSUMER_CLICKHOUSE_TOPIC: KAFKA_METRICS_CLICKHOUSE,
- METRICS_INGESTION_USE_PIPELINE_FRAMEWORK: false,
METRICS_REDIS_HOST: '127.0.0.1',
METRICS_REDIS_PORT: 6379,
METRICS_REDIS_PASSWORD: '',
diff --git a/nodejs/src/ingestion/pipelines/metrics/metrics-ingestion-consumer.test.ts b/nodejs/src/ingestion/pipelines/metrics/metrics-ingestion-consumer.test.ts
deleted file mode 100644
index 6e0f216b8d8f..000000000000
--- a/nodejs/src/ingestion/pipelines/metrics/metrics-ingestion-consumer.test.ts
+++ /dev/null
@@ -1,129 +0,0 @@
-import { mockProducer, mockProducerObserver } from '~/tests/helpers/mocks/producer.mock'
-
-import { Message } from 'node-rdkafka'
-
-import {
- KAFKA_APP_METRICS_2,
- KAFKA_METRICS_CLICKHOUSE,
- KAFKA_METRICS_INGESTION_DLQ,
-} from '~/common/config/kafka-topics'
-import { APP_METRICS_OUTPUT, AppMetricsOutput } from '~/common/outputs'
-import { IngestionOutputs } from '~/common/outputs/ingestion-outputs'
-import { SingleIngestionOutput } from '~/common/outputs/single-ingestion-output'
-import { closeHub, createHub } from '~/common/utils/db/hub'
-import { createTeam, createTestTeamFixture, getTeam } from '~/tests/helpers/sql'
-import { Hub, Team } from '~/types'
-
-import { getDefaultMetricsIngestionConsumerConfig } from './config'
-import { MetricsIngestionConsumer } from './metrics-ingestion-consumer'
-import {
- DEFAULT_METRICS_RETENTION_DAYS,
- METRICS_DLQ_OUTPUT,
- METRICS_OUTPUT,
- MetricsDlqOutput,
- MetricsOutput,
-} from './outputs/outputs'
-
-const createKafkaMessage = (headers: Record<string, string>): Message => {
- // The consumer passes the Avro payload through without decoding it.
- const value = Buffer.from('avro-payload')
- return {
- key: null,
- value,
- size: value.length,
- topic: 'test',
- offset: 0,
- timestamp: Date.now(),
- partition: 1,
- headers: Object.entries(headers).map(([key, headerValue]) => ({ [key]: Buffer.from(headerValue) })),
- }
-}
-
-describe('MetricsIngestionConsumer', () => {
- let consumer: MetricsIngestionConsumer
- let hub: Hub
- let team: Team
-
- beforeEach(async () => {
- hub = await createHub()
- team = (await createTestTeamFixture(hub.postgres)).team
-
- consumer = new MetricsIngestionConsumer(getDefaultMetricsIngestionConsumerConfig(), {
- teamManager: hub.teamManager,
- quotaLimiting: hub.quotaLimiting,
- outputs: new IngestionOutputs<MetricsOutput | MetricsDlqOutput | AppMetricsOutput>({
- [APP_METRICS_OUTPUT]: new SingleIngestionOutput(
- APP_METRICS_OUTPUT,
- KAFKA_APP_METRICS_2,
- mockProducer,
- 'test'
- ),
- [METRICS_OUTPUT]: new SingleIngestionOutput(
- METRICS_OUTPUT,
- KAFKA_METRICS_CLICKHOUSE,
- mockProducer,
- 'test'
- ),
- [METRICS_DLQ_OUTPUT]: new SingleIngestionOutput(
- METRICS_DLQ_OUTPUT,
- KAFKA_METRICS_INGESTION_DLQ,
- mockProducer,
- 'test'
- ),
- }),
- })
- // NOTE: We don't actually use kafka so we skip instantiation for faster tests
- consumer['kafkaConsumer'] = {
- connect: jest.fn(),
- disconnect: jest.fn(),
- isHealthy: jest.fn().mockReturnValue({ status: 'healthy' }),
- } as any
- await consumer.start()
-
- jest.spyOn(hub.quotaLimiting, 'isTeamTokenQuotaLimited').mockResolvedValue(false)
- })
-
- afterEach(async () => {
- await consumer.stop()
- await closeHub(hub)
- })
-
- it('marks a team that receives its first metrics as having ingested data', async () => {
- const newTeamId = await createTeam(hub.postgres, team.organization_id, undefined, { ingested_event: false })
- const newTeam = await getTeam(hub.postgres, newTeamId)
- const message = createKafkaMessage({
- token: newTeam!.api_token,
- bytes_uncompressed: '100',
- record_count: '1',
- })
-
- const { backgroundTask } = await consumer.processKafkaBatch([message])
- await backgroundTask
-
- expect((await getTeam(hub.postgres, newTeamId))?.ingested_event).toBe(true)
- })
-
- it('stamps the default retention-days header on messages produced to ClickHouse', async () => {
- const message = createKafkaMessage({
- token: team.api_token,
- bytes_uncompressed: '100',
- record_count: '1',
- })
-
- const { backgroundTask } = await consumer.processKafkaBatch([message])
- await backgroundTask
-
- const produced = mockProducerObserver
- .getProducedMessages()
- .filter((batch) => batch.topic === KAFKA_METRICS_CLICKHOUSE)
- expect(produced).toHaveLength(1)
- expect(produced[0].messages).toHaveLength(1)
- expect(produced[0].messages[0].headers).toEqual(
- expect.objectContaining({
- token: team.api_token,
- team_id: team.id.toString(),
- 'retention-days': DEFAULT_METRICS_RETENTION_DAYS.toString(),
- })
- )
- })
-})
diff --git a/nodejs/src/ingestion/pipelines/metrics/metrics-ingestion-consumer.ts b/nodejs/src/ingestion/pipelines/metrics/metrics-ingestion-consumer.ts
deleted file mode 100644
index 90b3b2d24da5..000000000000
--- a/nodejs/src/ingestion/pipelines/metrics/metrics-ingestion-consumer.ts
+++ /dev/null
@@ -1,418 +0,0 @@
-import { Message } from 'node-rdkafka'
-
-import { KafkaConsumerInterface, createKafkaConsumer, parseKafkaHeaders } from '~/common/kafka/consumer'
-import { AppMetricsOutput } from '~/common/outputs'
-import { IngestionOutputs } from '~/common/outputs/ingestion-outputs'
-import { RedisV2 } from '~/common/redis/redis-v2'
-import { AppMetricsAggregator } from '~/common/services/app-metrics-aggregator'
-import { QuotaLimiting } from '~/common/services/quota-limiting.service'
-import { instrumentFn, instrumented } from '~/common/tracing/tracing-utils'
-import { isDevEnv } from '~/common/utils/env-utils'
-import { logger } from '~/common/utils/logger'
-import { TeamManager } from '~/common/utils/team-manager'
-import { HealthCheckResult, PluginServerService } from '~/types'
-
-import { MetricsIngestionConsumerConfig } from './config'
-import { recordMetricsIngested } from './ingestion-otel-metrics'
-import {
- metricMessageDlqCounter,
- metricMessageDroppedCounter,
- metricsBytesAllowedCounter,
- metricsBytesDroppedCounter,
- metricsBytesReceivedCounter,
- metricsRecordsAllowedCounter,
- metricsRecordsDroppedCounter,
- metricsRecordsReceivedCounter,
-} from './metrics'
-import { DEFAULT_USAGE_STATS, UsageStatsByTeam } from './metrics-usage'
-import {
- DEFAULT_METRICS_RETENTION_DAYS,
- METRICS_DLQ_OUTPUT,
- METRICS_OUTPUT,
- MetricsDlqOutput,
- MetricsOutput,
-} from './outputs/outputs'
-import { MetricsRateLimiterService } from './services/metrics-rate-limiter.service'
-import { createMetricsRateLimiterRedis } from './services/metrics-redis'
-import { MetricsIngestionMessage } from './types'
-
-export interface MetricsIngestionConsumerDeps {
- teamManager: TeamManager
- quotaLimiting: QuotaLimiting
- /**
- * Resolved outputs registry — must include `METRICS_OUTPUT`, `METRICS_DLQ_OUTPUT`,
- * and `APP_METRICS_OUTPUT`. The producer + topic for each is wired by the
- * server via env vars — this consumer never touches a `KafkaProducerWrapper`
- * directly.
- */
- outputs: IngestionOutputs<MetricsOutput | MetricsDlqOutput | AppMetricsOutput>
-}
-
-/** Replaced by `MetricsPipelineConsumer`. Remove it when every environment sets `METRICS_INGESTION_USE_PIPELINE_FRAMEWORK`. */
-export class MetricsIngestionConsumer {
- protected name = 'MetricsIngestionConsumer'
- protected kafkaConsumer: KafkaConsumerInterface
- private appMetricsAggregator: AppMetricsAggregator
- private redis: RedisV2
- private rateLimiter: MetricsRateLimiterService
-
- protected groupId: string
- protected topic: string
-
- constructor(
- config: MetricsIngestionConsumerConfig,
- private deps: MetricsIngestionConsumerDeps,
- overrides: Partial<MetricsIngestionConsumerConfig> = {}
- ) {
- this.groupId = overrides.METRICS_INGESTION_CONSUMER_GROUP_ID ?? config.METRICS_INGESTION_CONSUMER_GROUP_ID
- this.topic =
- overrides.METRICS_INGESTION_CONSUMER_CONSUME_TOPIC ?? config.METRICS_INGESTION_CONSUMER_CONSUME_TOPIC
-
- this.appMetricsAggregator = new AppMetricsAggregator(deps.outputs)
-
- this.kafkaConsumer = createKafkaConsumer({ groupId: this.groupId, topic: this.topic })
- this.redis = createMetricsRateLimiterRedis({ ...config, ...overrides })
- this.rateLimiter = new MetricsRateLimiterService({ ...config, ...overrides }, this.redis)
- }
-
- public get service(): PluginServerService {
- return {
- id: this.name,
- onShutdown: async () => await this.stop(),
- healthcheck: () => this.isHealthy(),
- }
- }
-
- public async processBatch(
- messages: MetricsIngestionMessage[]
- ): Promise<{ backgroundTask?: Promise<any>; messages: MetricsIngestionMessage[] }> {
- if (!messages.length) {
- return { messages: [] }
- }
-
- this.trackIncomingTraffic(messages)
-
- const { quotaAllowedMessages, quotaDroppedMessages } = await this.filterQuotaLimitedMessages(messages)
- const { rateLimiterAllowedMessages, rateLimiterDroppedMessages } =
- await this.filterRateLimitedMessages(quotaAllowedMessages)
-
- const usageStats = this.trackOutgoingTrafficAndBuildUsageStats(rateLimiterAllowedMessages, [
- ...quotaDroppedMessages,
- ...rateLimiterDroppedMessages,
- ])
-
- return {
- backgroundTask: Promise.all([
- this.produceValidMetricMessages(rateLimiterAllowedMessages).then(() =>
- this.deps.teamManager.setTeamsIngestedData(
- rateLimiterAllowedMessages.map((message) => message.teamId)
- )
- ),
- this.emitUsageMetrics(usageStats),
- ]),
- messages: rateLimiterAllowedMessages,
- }
- }
-
- private trackIncomingTraffic(messages: MetricsIngestionMessage[]): void {
- let totalBytesReceived = 0
- let totalRecordsReceived = 0
- for (const message of messages) {
- totalBytesReceived += message.bytesUncompressed
- totalRecordsReceived += message.recordCount
- }
- metricsBytesReceivedCounter.inc(totalBytesReceived)
- metricsRecordsReceivedCounter.inc(totalRecordsReceived)
- }
-
- private trackOutgoingTrafficAndBuildUsageStats(
- allowedMessages: MetricsIngestionMessage[],
- droppedMessages: MetricsIngestionMessage[]
- ): UsageStatsByTeam {
- let totalBytesAllowed = 0
- let totalRecordsAllowed = 0
- const usageStats: UsageStatsByTeam = new Map()
-
- for (const message of allowedMessages) {
- const stats = usageStats.get(message.teamId) || { ...DEFAULT_USAGE_STATS }
- stats.bytesReceived += message.bytesUncompressed
- stats.recordsReceived += message.recordCount
- stats.bytesAllowed += message.bytesUncompressed
- stats.recordsAllowed += message.recordCount
- usageStats.set(message.teamId, stats)
-
- totalBytesAllowed += message.bytesUncompressed
- totalRecordsAllowed += message.recordCount
- }
-
- metricsBytesAllowedCounter.inc(totalBytesAllowed)
- metricsRecordsAllowedCounter.inc(totalRecordsAllowed)
-
- for (const message of droppedMessages) {
- const stats = usageStats.get(message.teamId) || { ...DEFAULT_USAGE_STATS }
- stats.bytesReceived += message.bytesUncompressed
- stats.recordsReceived += message.recordCount
- stats.bytesDropped += message.bytesUncompressed
- stats.recordsDropped += message.recordCount
- usageStats.set(message.teamId, stats)
- }
-
- for (const [teamId, stats] of usageStats) {
- const teamIdLabel = teamId.toString()
- if (stats.bytesDropped > 0) {
- metricsBytesDroppedCounter.inc({ team_id: teamIdLabel }, stats.bytesDropped)
- }
- if (stats.recordsDropped > 0) {
- metricsRecordsDroppedCounter.inc({ team_id: teamIdLabel }, stats.recordsDropped)
- }
- }
-
- return usageStats
- }
-
- private async filterQuotaLimitedMessages(messages: MetricsIngestionMessage[]): Promise<{
- quotaAllowedMessages: MetricsIngestionMessage[]
- quotaDroppedMessages: MetricsIngestionMessage[]
- }> {
- const quotaDroppedMessages: MetricsIngestionMessage[] = []
- const quotaAllowedMessages: MetricsIngestionMessage[] = []
-
- const uniqueTokens = [...new Set(messages.map((m) => m.token))]
-
- const quotaLimitedTokens = new Set(
- (
- await Promise.all(
- uniqueTokens.map(async (token) =>
- (await this.deps.quotaLimiting.isTeamTokenQuotaLimited(token, 'metrics_mb_ingested'))
- ? token
- : null
- )
- )
- ).filter((token): token is string => token !== null)
- )
-
- const droppedCountByTeam = new Map<number, number>()
- for (const message of messages) {
- if (quotaLimitedTokens.has(message.token)) {
- quotaDroppedMessages.push(message)
- droppedCountByTeam.set(message.teamId, (droppedCountByTeam.get(message.teamId) || 0) + 1)
- } else {
- quotaAllowedMessages.push(message)
- }
- }
-
- for (const [teamId, count] of droppedCountByTeam) {
- metricMessageDroppedCounter.inc({ reason: 'quota_limited', team_id: teamId.toString() }, count)
- }
-
- return { quotaAllowedMessages, quotaDroppedMessages }
- }
-
- private async filterRateLimitedMessages(messages: MetricsIngestionMessage[]): Promise<{
- rateLimiterAllowedMessages: MetricsIngestionMessage[]
- rateLimiterDroppedMessages: MetricsIngestionMessage[]
- }> {
- const { allowed, dropped } = await this.rateLimiter.filterMessages(messages)
-
- const droppedCountByTeam = new Map<number, number>()
- for (const message of dropped) {
- droppedCountByTeam.set(message.teamId, (droppedCountByTeam.get(message.teamId) || 0) + 1)
- }
- for (const [teamId, count] of droppedCountByTeam) {
- metricMessageDroppedCounter.inc({ reason: 'rate_limited', team_id: teamId.toString() }, count)
- }
-
- return { rateLimiterAllowedMessages: allowed, rateLimiterDroppedMessages: dropped }
- }
-
- private async produceValidMetricMessages(messages: MetricsIngestionMessage[]): Promise<void> {
- const results = await Promise.allSettled(
- messages.map(async (message) => {
- try {
- if (message.message.value === null) {
- return Promise.resolve()
- }
-
- // No enrichment needed for metrics — data is already structured numerically.
- // Just pass through the AVRO buffer as-is.
- await this.deps.outputs.queueMessages(METRICS_OUTPUT, [
- {
- value: message.message.value,
- key: null,
- headers: {
- ...parseKafkaHeaders(message.message.headers),
- token: message.token,
- team_id: message.teamId.toString(),
- 'retention-days': DEFAULT_METRICS_RETENTION_DAYS.toString(),
- },
- },
- ])
- // Only after the ClickHouse-bound produce resolves — messages that
- // fail and route to the DLQ must not count as ingested.
- recordMetricsIngested(message.teamId, message.bytesUncompressed, message.recordCount)
- } catch (error) {
- await this.produceToDlq(message, error)
- throw error
- }
- })
- )
-
- const failures = results.filter((r) => r.status === 'rejected')
- if (failures.length > 0) {
- logger.error('Failed to process some metric messages', {
- failureCount: failures.length,
- totalCount: messages.length,
- })
- }
- }
-
- private async produceToDlq(message: MetricsIngestionMessage, error: unknown): Promise<void> {
- const errorMessage = error instanceof Error ? error.message : String(error)
- const errorName = error instanceof Error ? error.name : 'UnknownError'
-
- metricMessageDlqCounter.inc({ reason: errorName, team_id: message.teamId.toString() })
-
- try {
- await this.deps.outputs.queueMessages(METRICS_DLQ_OUTPUT, [
- {
- value: message.message.value,
- key: null,
- headers: {
- ...parseKafkaHeaders(message.message.headers),
- token: message.token,
- team_id: message.teamId.toString(),
- error_message: errorMessage,
- error_name: errorName,
- failed_at: new Date().toISOString(),
- },
- },
- ])
- } catch (dlqError) {
- logger.error('Failed to produce message to DLQ', {
- error: dlqError,
- originalError: errorMessage,
- })
- }
- }
-
- private async emitUsageMetrics(usageStats: UsageStatsByTeam): Promise<void> {
- for (const [teamId, stats] of usageStats) {
- this.queueUsageMetric(teamId, 'bytes_received', stats.bytesReceived)
- this.queueUsageMetric(teamId, 'records_received', stats.recordsReceived)
- this.queueUsageMetric(teamId, 'bytes_ingested', stats.bytesAllowed)
- this.queueUsageMetric(teamId, 'records_ingested', stats.recordsAllowed)
- this.queueUsageMetric(teamId, 'bytes_dropped', stats.bytesDropped)
- this.queueUsageMetric(teamId, 'records_dropped', stats.recordsDropped)
- }
-
- // Best-effort: don't let metric failures block ingestion
- try {
- await this.appMetricsAggregator.flush()
- } catch (error) {
- logger.error('🔴', 'Failed to emit usage metrics - billing data may be lost', { error })
- }
- }
-
- private queueUsageMetric(teamId: number, metricName: string, count: number): void {
- if (count === 0) {
- return
- }
- this.appMetricsAggregator.queue({
- team_id: teamId,
- app_source: 'metrics',
- app_source_id: '',
- instance_id: '',
- metric_kind: 'usage',
- metric_name: metricName,
- count,
- })
- }
-
- @instrumented('metricsIngestionConsumer.handleEachBatch.parseKafkaMessages')
- public async _parseKafkaBatch(messages: Message[]): Promise<MetricsIngestionMessage[]> {
- const events: MetricsIngestionMessage[] = []
-
- await Promise.all(
- messages.map(async (message) => {
- try {
- const headers = parseKafkaHeaders(message.headers)
- const token = headers.token
-
- if (!token) {
- logger.error('missing_token')
- metricMessageDroppedCounter.inc({ reason: 'missing_token', team_id: 'unknown' })
- return
- }
-
- let team
- try {
- if (isDevEnv() && token === 'phc_local') {
- // phc_local is a special token used in dev to refer to team 1
- team = await this.deps.teamManager.getTeam(1)
- } else {
- team = await this.deps.teamManager.getTeamByToken(token)
- }
- } catch (e) {
- logger.error('team_lookup_error', { error: e })
- metricMessageDroppedCounter.inc({ reason: 'team_lookup_error', team_id: 'unknown' })
- return
- }
-
- if (!team) {
- logger.error('team_not_found', { token_with_no_team: token })
- metricMessageDroppedCounter.inc({ reason: 'team_not_found', team_id: 'unknown' })
- return
- }
-
- const bytesUncompressed = parseInt(headers.bytes_uncompressed ?? '0', 10)
- const bytesCompressed = parseInt(headers.bytes_compressed ?? '0', 10)
- const recordCount = parseInt(headers.record_count ?? '0', 10)
-
- events.push({
- token,
- message,
- teamId: team.id,
- bytesUncompressed,
- bytesCompressed,
- recordCount,
- })
- } catch (e) {
- logger.error('Error parsing message', e)
- metricMessageDroppedCounter.inc({ reason: 'parse_error', team_id: 'unknown' })
- return
- }
- })
- )
-
- return events
- }
-
- public async processKafkaBatch(
- messages: Message[]
- ): Promise<{ backgroundTask?: Promise<any>; messages: MetricsIngestionMessage[] }> {
- const events = await this._parseKafkaBatch(messages)
- return await this.processBatch(events)
- }
-
- public async start(): Promise<void> {
- await this.kafkaConsumer.connect(async (messages) => {
- logger.info('🔁', `${this.name} - handling batch`, {
- size: messages.length,
- })
-
- return await instrumentFn('metricsIngestionConsumer.handleEachBatch', async () => {
- return await this.processKafkaBatch(messages)
- })
- })
- }
-
- public async stop(): Promise<void> {
- logger.info('💤', 'Stopping metrics consumer...')
- await this.kafkaConsumer.disconnect()
- logger.info('💤', 'Metrics consumer stopped!')
- }
-
- public isHealthy(): HealthCheckResult {
- return this.kafkaConsumer.isHealthy()
- }
-}
diff --git a/nodejs/src/ingestion/pipelines/metrics/metrics-pipeline-consumer.ts b/nodejs/src/ingestion/pipelines/metrics/metrics-pipeline-consumer.ts
index 3a79028cc233..bee8ebc90881 100644
--- a/nodejs/src/ingestion/pipelines/metrics/metrics-pipeline-consumer.ts
+++ b/nodejs/src/ingestion/pipelines/metrics/metrics-pipeline-consumer.ts
@@ -53,7 +53,7 @@ class FailureLatchingPromiseScheduler extends PromiseScheduler {
}
export class MetricsPipelineConsumer {
- // Same id as the pre-framework consumer, so health checks keep their name when the flag changes.
+ // Health checks and dashboards key on this service id.
protected name = 'MetricsIngestionConsumer'
protected kafkaConsumer: KafkaConsumerInterface
private promiseScheduler: FailureLatchingPromiseScheduler
diff --git a/nodejs/src/ingestion/pipelines/metrics/metrics-usage.ts b/nodejs/src/ingestion/pipelines/metrics/metrics-usage.ts
index 1c173da4bdad..5505805fa415 100644
--- a/nodejs/src/ingestion/pipelines/metrics/metrics-usage.ts
+++ b/nodejs/src/ingestion/pipelines/metrics/metrics-usage.ts
@@ -7,7 +7,7 @@ export type UsageStats = {
recordsDropped: number
}
-export const DEFAULT_USAGE_STATS: UsageStats = {
+const DEFAULT_USAGE_STATS: UsageStats = {
bytesReceived: 0,
recordsReceived: 0,
bytesAllowed: 0,
@@ -16,7 +16,7 @@ export const DEFAULT_USAGE_STATS: UsageStats = {
recordsDropped: 0,
}
-export type UsageStatsByTeam = Map<number, UsageStats>
+type UsageStatsByTeam = Map<number, UsageStats>
export class MetricsUsageAccumulator {
private readonly stats: UsageStatsByTeam = new Map()
diff --git a/nodejs/src/ingestion/pipelines/metrics/outputs/outputs.ts b/nodejs/src/ingestion/pipelines/metrics/outputs/outputs.ts
index a09c3d0a14fa..618b2af5834c 100644
--- a/nodejs/src/ingestion/pipelines/metrics/outputs/outputs.ts
+++ b/nodejs/src/ingestion/pipelines/metrics/outputs/outputs.ts
@@ -5,9 +5,5 @@
export const METRICS_OUTPUT = 'metrics' as const
export type MetricsOutput = typeof METRICS_OUTPUT
-/** Used only by the pre-framework consumer. */
-export const METRICS_DLQ_OUTPUT = 'metrics_dlq' as const
-export type MetricsDlqOutput = typeof METRICS_DLQ_OUTPUT
-
/** Metrics has no per-team retention setting. ClickHouse applies its own default only when the `retention-days` header is missing. */
export const DEFAULT_METRICS_RETENTION_DAYS = 30
diff --git a/nodejs/src/ingestion/pipelines/metrics/outputs/registry.ts b/nodejs/src/ingestion/pipelines/metrics/outputs/registry.ts
index 5fb62c1f761f..6c28b7de2ab7 100644
--- a/nodejs/src/ingestion/pipelines/metrics/outputs/registry.ts
+++ b/nodejs/src/ingestion/pipelines/metrics/outputs/registry.ts
@@ -1,14 +1,13 @@
import { APP_METRICS_OUTPUT, DLQ_OUTPUT } from '~/common/outputs'
import { IngestionOutputsBuilder } from '~/common/outputs/ingestion-outputs-builder'
-import { METRICS_DLQ_OUTPUT, METRICS_OUTPUT } from './outputs'
+import { METRICS_OUTPUT } from './outputs'
/**
* Outputs for the metrics ingestion deployment.
*
* - `METRICS_OUTPUT` — main metrics data path → topic from `METRICS_INGESTION_CONSUMER_CLICKHOUSE_TOPIC`.
* - `DLQ_OUTPUT` — DLQ for failed messages → topic from `METRICS_INGESTION_CONSUMER_DLQ_TOPIC`.
- * - `METRICS_DLQ_OUTPUT` — the same topic, for the pre-framework consumer.
* - `APP_METRICS_OUTPUT` — usage metrics → topic from `METRICS_INGESTION_OUTPUT_APP_METRICS_TOPIC`.
*
* Per-output producer is env-controlled (`*_PRODUCER` keys) so the route can be
@@ -28,8 +27,4 @@ export function createMetricsOutputsRegistry() {
topicKey: 'METRICS_INGESTION_CONSUMER_DLQ_TOPIC',
producerKey: 'METRICS_INGESTION_OUTPUT_DLQ_PRODUCER',
})
- .register(METRICS_DLQ_OUTPUT, {
- topicKey: 'METRICS_INGESTION_CONSUMER_DLQ_TOPIC',
- producerKey: 'METRICS_INGESTION_OUTPUT_DLQ_PRODUCER',
- })
}
diff --git a/nodejs/src/servers/ingestion-metrics-server.ts b/nodejs/src/servers/ingestion-metrics-server.ts
index 7b1b2fe913c0..80af1f9f606e 100644
--- a/nodejs/src/servers/ingestion-metrics-server.ts
+++ b/nodejs/src/servers/ingestion-metrics-server.ts
@@ -12,7 +12,6 @@ import {
getDefaultMetricsIngestionConsumerConfig,
getDefaultMetricsIngestionOutputsConfig,
} from '~/ingestion/pipelines/metrics/config'
-import { MetricsIngestionConsumer } from '~/ingestion/pipelines/metrics/metrics-ingestion-consumer'
import { MetricsPipelineConsumer } from '~/ingestion/pipelines/metrics/metrics-pipeline-consumer'
import { createProducerRegistry } from '~/ingestion/pipelines/metrics/outputs/producer-registry'
import {
@@ -110,10 +109,7 @@ export class IngestionMetricsServer implements NodeServer {
const serviceLoaders: (() => Promise<PluginServerService>)[] = []
serviceLoaders.push(async () => {
- const deps = { teamManager, quotaLimiting, outputs }
- const consumer = this.config.METRICS_INGESTION_USE_PIPELINE_FRAMEWORK
- ? new MetricsPipelineConsumer(this.config, deps)
- : new MetricsIngestionConsumer(this.config, deps)
+ const consumer = new MetricsPipelineConsumer(this.config, { teamManager, quotaLimiting, outputs })
await consumer.start()
return consumer.service
})