:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
chore(ingestion): remove legacy metrics ingestion consumer (#128375)
Frank Hamand2f59876c951b
diff --git a/nodejs/src/ingestion/pipelines/metrics/config.ts b/nodejs/src/ingestion/pipelines/metrics/config.tsindex 7fc7d098b800..9d38b663f577 100644--- a/nodejs/src/ingestion/pipelines/metrics/config.ts+++ b/nodejs/src/ingestion/pipelines/metrics/config.ts@@ -29,7 +29,6 @@ export type MetricsIngestionConsumerConfig = {METRICS_INGESTION_CONSUMER_CONSUME_TOPIC: stringMETRICS_INGESTION_CONSUMER_DLQ_TOPIC: stringMETRICS_INGESTION_CONSUMER_CLICKHOUSE_TOPIC: string- METRICS_INGESTION_USE_PIPELINE_FRAMEWORK: booleanMETRICS_REDIS_HOST: stringMETRICS_REDIS_PORT: numberMETRICS_REDIS_PASSWORD: string@@ -53,7 +52,6 @@ export function getDefaultMetricsIngestionConsumerConfig(): MetricsIngestionConsMETRICS_INGESTION_CONSUMER_CONSUME_TOPIC: KAFKA_METRICS_INGESTION,METRICS_INGESTION_CONSUMER_DLQ_TOPIC: KAFKA_METRICS_INGESTION_DLQ,METRICS_INGESTION_CONSUMER_CLICKHOUSE_TOPIC: KAFKA_METRICS_CLICKHOUSE,- METRICS_INGESTION_USE_PIPELINE_FRAMEWORK: false,METRICS_REDIS_HOST: '127.0.0.1',METRICS_REDIS_PORT: 6379,METRICS_REDIS_PASSWORD: '',diff --git a/nodejs/src/ingestion/pipelines/metrics/metrics-ingestion-consumer.test.ts b/nodejs/src/ingestion/pipelines/metrics/metrics-ingestion-consumer.test.tsdeleted file mode 100644index 6e0f216b8d8f..000000000000--- a/nodejs/src/ingestion/pipelines/metrics/metrics-ingestion-consumer.test.ts+++ /dev/null@@ -1,129 +0,0 @@-import { mockProducer, mockProducerObserver } from '~/tests/helpers/mocks/producer.mock'--import { Message } from 'node-rdkafka'--import {- KAFKA_APP_METRICS_2,- KAFKA_METRICS_CLICKHOUSE,- KAFKA_METRICS_INGESTION_DLQ,-} from '~/common/config/kafka-topics'-import { APP_METRICS_OUTPUT, AppMetricsOutput } from '~/common/outputs'-import { IngestionOutputs } from '~/common/outputs/ingestion-outputs'-import { SingleIngestionOutput } from '~/common/outputs/single-ingestion-output'-import { closeHub, createHub } from '~/common/utils/db/hub'-import { createTeam, createTestTeamFixture, getTeam } from '~/tests/helpers/sql'-import { Hub, Team } from '~/types'--import { getDefaultMetricsIngestionConsumerConfig } from './config'-import { MetricsIngestionConsumer } from './metrics-ingestion-consumer'-import {- DEFAULT_METRICS_RETENTION_DAYS,- METRICS_DLQ_OUTPUT,- METRICS_OUTPUT,- MetricsDlqOutput,- MetricsOutput,-} from './outputs/outputs'--const createKafkaMessage = (headers: Record<string, string>): Message => {- // The consumer passes the Avro payload through without decoding it.- const value = Buffer.from('avro-payload')- return {- key: null,- value,- size: value.length,- topic: 'test',- offset: 0,- timestamp: Date.now(),- partition: 1,- headers: Object.entries(headers).map(([key, headerValue]) => ({ [key]: Buffer.from(headerValue) })),- }-}--describe('MetricsIngestionConsumer', () => {- let consumer: MetricsIngestionConsumer- let hub: Hub- let team: Team-- beforeEach(async () => {- hub = await createHub()- team = (await createTestTeamFixture(hub.postgres)).team-- consumer = new MetricsIngestionConsumer(getDefaultMetricsIngestionConsumerConfig(), {- teamManager: hub.teamManager,- quotaLimiting: hub.quotaLimiting,- outputs: new IngestionOutputs<MetricsOutput | MetricsDlqOutput | AppMetricsOutput>({- [APP_METRICS_OUTPUT]: new SingleIngestionOutput(- APP_METRICS_OUTPUT,- KAFKA_APP_METRICS_2,- mockProducer,- 'test'- ),- [METRICS_OUTPUT]: new SingleIngestionOutput(- METRICS_OUTPUT,- KAFKA_METRICS_CLICKHOUSE,- mockProducer,- 'test'- ),- [METRICS_DLQ_OUTPUT]: new SingleIngestionOutput(- METRICS_DLQ_OUTPUT,- KAFKA_METRICS_INGESTION_DLQ,- mockProducer,- 'test'- ),- }),- })- // NOTE: We don't actually use kafka so we skip instantiation for faster tests- consumer['kafkaConsumer'] = {- connect: jest.fn(),- disconnect: jest.fn(),- isHealthy: jest.fn().mockReturnValue({ status: 'healthy' }),- } as any- await consumer.start()-- jest.spyOn(hub.quotaLimiting, 'isTeamTokenQuotaLimited').mockResolvedValue(false)- })-- afterEach(async () => {- await consumer.stop()- await closeHub(hub)- })-- it('marks a team that receives its first metrics as having ingested data', async () => {- const newTeamId = await createTeam(hub.postgres, team.organization_id, undefined, { ingested_event: false })- const newTeam = await getTeam(hub.postgres, newTeamId)- const message = createKafkaMessage({- token: newTeam!.api_token,- bytes_uncompressed: '100',- record_count: '1',- })-- const { backgroundTask } = await consumer.processKafkaBatch([message])- await backgroundTask-- expect((await getTeam(hub.postgres, newTeamId))?.ingested_event).toBe(true)- })-- it('stamps the default retention-days header on messages produced to ClickHouse', async () => {- const message = createKafkaMessage({- token: team.api_token,- bytes_uncompressed: '100',- record_count: '1',- })-- const { backgroundTask } = await consumer.processKafkaBatch([message])- await backgroundTask-- const produced = mockProducerObserver- .getProducedMessages()- .filter((batch) => batch.topic === KAFKA_METRICS_CLICKHOUSE)- expect(produced).toHaveLength(1)- expect(produced[0].messages).toHaveLength(1)- expect(produced[0].messages[0].headers).toEqual(- expect.objectContaining({- token: team.api_token,- team_id: team.id.toString(),- 'retention-days': DEFAULT_METRICS_RETENTION_DAYS.toString(),- })- )- })-})diff --git a/nodejs/src/ingestion/pipelines/metrics/metrics-ingestion-consumer.ts b/nodejs/src/ingestion/pipelines/metrics/metrics-ingestion-consumer.tsdeleted file mode 100644index 90b3b2d24da5..000000000000--- a/nodejs/src/ingestion/pipelines/metrics/metrics-ingestion-consumer.ts+++ /dev/null@@ -1,418 +0,0 @@-import { Message } from 'node-rdkafka'--import { KafkaConsumerInterface, createKafkaConsumer, parseKafkaHeaders } from '~/common/kafka/consumer'-import { AppMetricsOutput } from '~/common/outputs'-import { IngestionOutputs } from '~/common/outputs/ingestion-outputs'-import { RedisV2 } from '~/common/redis/redis-v2'-import { AppMetricsAggregator } from '~/common/services/app-metrics-aggregator'-import { QuotaLimiting } from '~/common/services/quota-limiting.service'-import { instrumentFn, instrumented } from '~/common/tracing/tracing-utils'-import { isDevEnv } from '~/common/utils/env-utils'-import { logger } from '~/common/utils/logger'-import { TeamManager } from '~/common/utils/team-manager'-import { HealthCheckResult, PluginServerService } from '~/types'--import { MetricsIngestionConsumerConfig } from './config'-import { recordMetricsIngested } from './ingestion-otel-metrics'-import {- metricMessageDlqCounter,- metricMessageDroppedCounter,- metricsBytesAllowedCounter,- metricsBytesDroppedCounter,- metricsBytesReceivedCounter,- metricsRecordsAllowedCounter,- metricsRecordsDroppedCounter,- metricsRecordsReceivedCounter,-} from './metrics'-import { DEFAULT_USAGE_STATS, UsageStatsByTeam } from './metrics-usage'-import {- DEFAULT_METRICS_RETENTION_DAYS,- METRICS_DLQ_OUTPUT,- METRICS_OUTPUT,- MetricsDlqOutput,- MetricsOutput,-} from './outputs/outputs'-import { MetricsRateLimiterService } from './services/metrics-rate-limiter.service'-import { createMetricsRateLimiterRedis } from './services/metrics-redis'-import { MetricsIngestionMessage } from './types'--export interface MetricsIngestionConsumerDeps {- teamManager: TeamManager- quotaLimiting: QuotaLimiting- /**- * Resolved outputs registry — must include `METRICS_OUTPUT`, `METRICS_DLQ_OUTPUT`,- * and `APP_METRICS_OUTPUT`. The producer + topic for each is wired by the- * server via env vars — this consumer never touches a `KafkaProducerWrapper`- * directly.- */- outputs: IngestionOutputs<MetricsOutput | MetricsDlqOutput | AppMetricsOutput>-}--/** Replaced by `MetricsPipelineConsumer`. Remove it when every environment sets `METRICS_INGESTION_USE_PIPELINE_FRAMEWORK`. */-export class MetricsIngestionConsumer {- protected name = 'MetricsIngestionConsumer'- protected kafkaConsumer: KafkaConsumerInterface- private appMetricsAggregator: AppMetricsAggregator- private redis: RedisV2- private rateLimiter: MetricsRateLimiterService-- protected groupId: string- protected topic: string-- constructor(- config: MetricsIngestionConsumerConfig,- private deps: MetricsIngestionConsumerDeps,- overrides: Partial<MetricsIngestionConsumerConfig> = {}- ) {- this.groupId = overrides.METRICS_INGESTION_CONSUMER_GROUP_ID ?? config.METRICS_INGESTION_CONSUMER_GROUP_ID- this.topic =- overrides.METRICS_INGESTION_CONSUMER_CONSUME_TOPIC ?? config.METRICS_INGESTION_CONSUMER_CONSUME_TOPIC-- this.appMetricsAggregator = new AppMetricsAggregator(deps.outputs)-- this.kafkaConsumer = createKafkaConsumer({ groupId: this.groupId, topic: this.topic })- this.redis = createMetricsRateLimiterRedis({ ...config, ...overrides })- this.rateLimiter = new MetricsRateLimiterService({ ...config, ...overrides }, this.redis)- }-- public get service(): PluginServerService {- return {- id: this.name,- onShutdown: async () => await this.stop(),- healthcheck: () => this.isHealthy(),- }- }-- public async processBatch(- messages: MetricsIngestionMessage[]- ): Promise<{ backgroundTask?: Promise<any>; messages: MetricsIngestionMessage[] }> {- if (!messages.length) {- return { messages: [] }- }-- this.trackIncomingTraffic(messages)-- const { quotaAllowedMessages, quotaDroppedMessages } = await this.filterQuotaLimitedMessages(messages)- const { rateLimiterAllowedMessages, rateLimiterDroppedMessages } =- await this.filterRateLimitedMessages(quotaAllowedMessages)-- const usageStats = this.trackOutgoingTrafficAndBuildUsageStats(rateLimiterAllowedMessages, [- ...quotaDroppedMessages,- ...rateLimiterDroppedMessages,- ])-- return {- backgroundTask: Promise.all([- this.produceValidMetricMessages(rateLimiterAllowedMessages).then(() =>- this.deps.teamManager.setTeamsIngestedData(- rateLimiterAllowedMessages.map((message) => message.teamId)- )- ),- this.emitUsageMetrics(usageStats),- ]),- messages: rateLimiterAllowedMessages,- }- }-- private trackIncomingTraffic(messages: MetricsIngestionMessage[]): void {- let totalBytesReceived = 0- let totalRecordsReceived = 0- for (const message of messages) {- totalBytesReceived += message.bytesUncompressed- totalRecordsReceived += message.recordCount- }- metricsBytesReceivedCounter.inc(totalBytesReceived)- metricsRecordsReceivedCounter.inc(totalRecordsReceived)- }-- private trackOutgoingTrafficAndBuildUsageStats(- allowedMessages: MetricsIngestionMessage[],- droppedMessages: MetricsIngestionMessage[]- ): UsageStatsByTeam {- let totalBytesAllowed = 0- let totalRecordsAllowed = 0- const usageStats: UsageStatsByTeam = new Map()-- for (const message of allowedMessages) {- const stats = usageStats.get(message.teamId) || { ...DEFAULT_USAGE_STATS }- stats.bytesReceived += message.bytesUncompressed- stats.recordsReceived += message.recordCount- stats.bytesAllowed += message.bytesUncompressed- stats.recordsAllowed += message.recordCount- usageStats.set(message.teamId, stats)-- totalBytesAllowed += message.bytesUncompressed- totalRecordsAllowed += message.recordCount- }-- metricsBytesAllowedCounter.inc(totalBytesAllowed)- metricsRecordsAllowedCounter.inc(totalRecordsAllowed)-- for (const message of droppedMessages) {- const stats = usageStats.get(message.teamId) || { ...DEFAULT_USAGE_STATS }- stats.bytesReceived += message.bytesUncompressed- stats.recordsReceived += message.recordCount- stats.bytesDropped += message.bytesUncompressed- stats.recordsDropped += message.recordCount- usageStats.set(message.teamId, stats)- }-- for (const [teamId, stats] of usageStats) {- const teamIdLabel = teamId.toString()- if (stats.bytesDropped > 0) {- metricsBytesDroppedCounter.inc({ team_id: teamIdLabel }, stats.bytesDropped)- }- if (stats.recordsDropped > 0) {- metricsRecordsDroppedCounter.inc({ team_id: teamIdLabel }, stats.recordsDropped)- }- }-- return usageStats- }-- private async filterQuotaLimitedMessages(messages: MetricsIngestionMessage[]): Promise<{- quotaAllowedMessages: MetricsIngestionMessage[]- quotaDroppedMessages: MetricsIngestionMessage[]- }> {- const quotaDroppedMessages: MetricsIngestionMessage[] = []- const quotaAllowedMessages: MetricsIngestionMessage[] = []-- const uniqueTokens = [...new Set(messages.map((m) => m.token))]-- const quotaLimitedTokens = new Set(- (- await Promise.all(- uniqueTokens.map(async (token) =>- (await this.deps.quotaLimiting.isTeamTokenQuotaLimited(token, 'metrics_mb_ingested'))- ? token- : null- )- )- ).filter((token): token is string => token !== null)- )-- const droppedCountByTeam = new Map<number, number>()- for (const message of messages) {- if (quotaLimitedTokens.has(message.token)) {- quotaDroppedMessages.push(message)- droppedCountByTeam.set(message.teamId, (droppedCountByTeam.get(message.teamId) || 0) + 1)- } else {- quotaAllowedMessages.push(message)- }- }-- for (const [teamId, count] of droppedCountByTeam) {- metricMessageDroppedCounter.inc({ reason: 'quota_limited', team_id: teamId.toString() }, count)- }-- return { quotaAllowedMessages, quotaDroppedMessages }- }-- private async filterRateLimitedMessages(messages: MetricsIngestionMessage[]): Promise<{- rateLimiterAllowedMessages: MetricsIngestionMessage[]- rateLimiterDroppedMessages: MetricsIngestionMessage[]- }> {- const { allowed, dropped } = await this.rateLimiter.filterMessages(messages)-- const droppedCountByTeam = new Map<number, number>()- for (const message of dropped) {- droppedCountByTeam.set(message.teamId, (droppedCountByTeam.get(message.teamId) || 0) + 1)- }- for (const [teamId, count] of droppedCountByTeam) {- metricMessageDroppedCounter.inc({ reason: 'rate_limited', team_id: teamId.toString() }, count)- }-- return { rateLimiterAllowedMessages: allowed, rateLimiterDroppedMessages: dropped }- }-- private async produceValidMetricMessages(messages: MetricsIngestionMessage[]): Promise<void> {- const results = await Promise.allSettled(- messages.map(async (message) => {- try {- if (message.message.value === null) {- return Promise.resolve()- }-- // No enrichment needed for metrics — data is already structured numerically.- // Just pass through the AVRO buffer as-is.- await this.deps.outputs.queueMessages(METRICS_OUTPUT, [- {- value: message.message.value,- key: null,- headers: {- ...parseKafkaHeaders(message.message.headers),- token: message.token,- team_id: message.teamId.toString(),- 'retention-days': DEFAULT_METRICS_RETENTION_DAYS.toString(),- },- },- ])- // Only after the ClickHouse-bound produce resolves — messages that- // fail and route to the DLQ must not count as ingested.- recordMetricsIngested(message.teamId, message.bytesUncompressed, message.recordCount)- } catch (error) {- await this.produceToDlq(message, error)- throw error- }- })- )-- const failures = results.filter((r) => r.status === 'rejected')- if (failures.length > 0) {- logger.error('Failed to process some metric messages', {- failureCount: failures.length,- totalCount: messages.length,- })- }- }-- private async produceToDlq(message: MetricsIngestionMessage, error: unknown): Promise<void> {- const errorMessage = error instanceof Error ? error.message : String(error)- const errorName = error instanceof Error ? error.name : 'UnknownError'-- metricMessageDlqCounter.inc({ reason: errorName, team_id: message.teamId.toString() })-- try {- await this.deps.outputs.queueMessages(METRICS_DLQ_OUTPUT, [- {- value: message.message.value,- key: null,- headers: {- ...parseKafkaHeaders(message.message.headers),- token: message.token,- team_id: message.teamId.toString(),- error_message: errorMessage,- error_name: errorName,- failed_at: new Date().toISOString(),- },- },- ])- } catch (dlqError) {- logger.error('Failed to produce message to DLQ', {- error: dlqError,- originalError: errorMessage,- })- }- }-- private async emitUsageMetrics(usageStats: UsageStatsByTeam): Promise<void> {- for (const [teamId, stats] of usageStats) {- this.queueUsageMetric(teamId, 'bytes_received', stats.bytesReceived)- this.queueUsageMetric(teamId, 'records_received', stats.recordsReceived)- this.queueUsageMetric(teamId, 'bytes_ingested', stats.bytesAllowed)- this.queueUsageMetric(teamId, 'records_ingested', stats.recordsAllowed)- this.queueUsageMetric(teamId, 'bytes_dropped', stats.bytesDropped)- this.queueUsageMetric(teamId, 'records_dropped', stats.recordsDropped)- }-- // Best-effort: don't let metric failures block ingestion- try {- await this.appMetricsAggregator.flush()- } catch (error) {- logger.error('🔴', 'Failed to emit usage metrics - billing data may be lost', { error })- }- }-- private queueUsageMetric(teamId: number, metricName: string, count: number): void {- if (count === 0) {- return- }- this.appMetricsAggregator.queue({- team_id: teamId,- app_source: 'metrics',- app_source_id: '',- instance_id: '',- metric_kind: 'usage',- metric_name: metricName,- count,- })- }-- @instrumented('metricsIngestionConsumer.handleEachBatch.parseKafkaMessages')- public async _parseKafkaBatch(messages: Message[]): Promise<MetricsIngestionMessage[]> {- const events: MetricsIngestionMessage[] = []-- await Promise.all(- messages.map(async (message) => {- try {- const headers = parseKafkaHeaders(message.headers)- const token = headers.token-- if (!token) {- logger.error('missing_token')- metricMessageDroppedCounter.inc({ reason: 'missing_token', team_id: 'unknown' })- return- }-- let team- try {- if (isDevEnv() && token === 'phc_local') {- // phc_local is a special token used in dev to refer to team 1- team = await this.deps.teamManager.getTeam(1)- } else {- team = await this.deps.teamManager.getTeamByToken(token)- }- } catch (e) {- logger.error('team_lookup_error', { error: e })- metricMessageDroppedCounter.inc({ reason: 'team_lookup_error', team_id: 'unknown' })- return- }-- if (!team) {- logger.error('team_not_found', { token_with_no_team: token })- metricMessageDroppedCounter.inc({ reason: 'team_not_found', team_id: 'unknown' })- return- }-- const bytesUncompressed = parseInt(headers.bytes_uncompressed ?? '0', 10)- const bytesCompressed = parseInt(headers.bytes_compressed ?? '0', 10)- const recordCount = parseInt(headers.record_count ?? '0', 10)-- events.push({- token,- message,- teamId: team.id,- bytesUncompressed,- bytesCompressed,- recordCount,- })- } catch (e) {- logger.error('Error parsing message', e)- metricMessageDroppedCounter.inc({ reason: 'parse_error', team_id: 'unknown' })- return- }- })- )-- return events- }-- public async processKafkaBatch(- messages: Message[]- ): Promise<{ backgroundTask?: Promise<any>; messages: MetricsIngestionMessage[] }> {- const events = await this._parseKafkaBatch(messages)- return await this.processBatch(events)- }-- public async start(): Promise<void> {- await this.kafkaConsumer.connect(async (messages) => {- logger.info('🔁', `${this.name} - handling batch`, {- size: messages.length,- })-- return await instrumentFn('metricsIngestionConsumer.handleEachBatch', async () => {- return await this.processKafkaBatch(messages)- })- })- }-- public async stop(): Promise<void> {- logger.info('💤', 'Stopping metrics consumer...')- await this.kafkaConsumer.disconnect()- logger.info('💤', 'Metrics consumer stopped!')- }-- public isHealthy(): HealthCheckResult {- return this.kafkaConsumer.isHealthy()- }-}diff --git a/nodejs/src/ingestion/pipelines/metrics/metrics-pipeline-consumer.ts b/nodejs/src/ingestion/pipelines/metrics/metrics-pipeline-consumer.tsindex 3a79028cc233..bee8ebc90881 100644--- a/nodejs/src/ingestion/pipelines/metrics/metrics-pipeline-consumer.ts+++ b/nodejs/src/ingestion/pipelines/metrics/metrics-pipeline-consumer.ts@@ -53,7 +53,7 @@ class FailureLatchingPromiseScheduler extends PromiseScheduler {}export class MetricsPipelineConsumer {- // Same id as the pre-framework consumer, so health checks keep their name when the flag changes.+ // Health checks and dashboards key on this service id.protected name = 'MetricsIngestionConsumer'protected kafkaConsumer: KafkaConsumerInterfaceprivate promiseScheduler: FailureLatchingPromiseSchedulerdiff --git a/nodejs/src/ingestion/pipelines/metrics/metrics-usage.ts b/nodejs/src/ingestion/pipelines/metrics/metrics-usage.tsindex 1c173da4bdad..5505805fa415 100644--- a/nodejs/src/ingestion/pipelines/metrics/metrics-usage.ts+++ b/nodejs/src/ingestion/pipelines/metrics/metrics-usage.ts@@ -7,7 +7,7 @@ export type UsageStats = {recordsDropped: number}-export const DEFAULT_USAGE_STATS: UsageStats = {+const DEFAULT_USAGE_STATS: UsageStats = {bytesReceived: 0,recordsReceived: 0,bytesAllowed: 0,@@ -16,7 +16,7 @@ export const DEFAULT_USAGE_STATS: UsageStats = {recordsDropped: 0,}-export type UsageStatsByTeam = Map<number, UsageStats>+type UsageStatsByTeam = Map<number, UsageStats>export class MetricsUsageAccumulator {private readonly stats: UsageStatsByTeam = new Map()diff --git a/nodejs/src/ingestion/pipelines/metrics/outputs/outputs.ts b/nodejs/src/ingestion/pipelines/metrics/outputs/outputs.tsindex a09c3d0a14fa..618b2af5834c 100644--- a/nodejs/src/ingestion/pipelines/metrics/outputs/outputs.ts+++ b/nodejs/src/ingestion/pipelines/metrics/outputs/outputs.ts@@ -5,9 +5,5 @@export const METRICS_OUTPUT = 'metrics' as constexport type MetricsOutput = typeof METRICS_OUTPUT-/** Used only by the pre-framework consumer. */-export const METRICS_DLQ_OUTPUT = 'metrics_dlq' as const-export type MetricsDlqOutput = typeof METRICS_DLQ_OUTPUT-/** Metrics has no per-team retention setting. ClickHouse applies its own default only when the `retention-days` header is missing. */export const DEFAULT_METRICS_RETENTION_DAYS = 30diff --git a/nodejs/src/ingestion/pipelines/metrics/outputs/registry.ts b/nodejs/src/ingestion/pipelines/metrics/outputs/registry.tsindex 5fb62c1f761f..6c28b7de2ab7 100644--- a/nodejs/src/ingestion/pipelines/metrics/outputs/registry.ts+++ b/nodejs/src/ingestion/pipelines/metrics/outputs/registry.ts@@ -1,14 +1,13 @@import { APP_METRICS_OUTPUT, DLQ_OUTPUT } from '~/common/outputs'import { IngestionOutputsBuilder } from '~/common/outputs/ingestion-outputs-builder'-import { METRICS_DLQ_OUTPUT, METRICS_OUTPUT } from './outputs'+import { METRICS_OUTPUT } from './outputs'/*** Outputs for the metrics ingestion deployment.** - `METRICS_OUTPUT` — main metrics data path → topic from `METRICS_INGESTION_CONSUMER_CLICKHOUSE_TOPIC`.* - `DLQ_OUTPUT` — DLQ for failed messages → topic from `METRICS_INGESTION_CONSUMER_DLQ_TOPIC`.- * - `METRICS_DLQ_OUTPUT` — the same topic, for the pre-framework consumer.* - `APP_METRICS_OUTPUT` — usage metrics → topic from `METRICS_INGESTION_OUTPUT_APP_METRICS_TOPIC`.** Per-output producer is env-controlled (`*_PRODUCER` keys) so the route can be@@ -28,8 +27,4 @@ export function createMetricsOutputsRegistry() {topicKey: 'METRICS_INGESTION_CONSUMER_DLQ_TOPIC',producerKey: 'METRICS_INGESTION_OUTPUT_DLQ_PRODUCER',})- .register(METRICS_DLQ_OUTPUT, {- topicKey: 'METRICS_INGESTION_CONSUMER_DLQ_TOPIC',- producerKey: 'METRICS_INGESTION_OUTPUT_DLQ_PRODUCER',- })}diff --git a/nodejs/src/servers/ingestion-metrics-server.ts b/nodejs/src/servers/ingestion-metrics-server.tsindex 7b1b2fe913c0..80af1f9f606e 100644--- a/nodejs/src/servers/ingestion-metrics-server.ts+++ b/nodejs/src/servers/ingestion-metrics-server.ts@@ -12,7 +12,6 @@ import {getDefaultMetricsIngestionConsumerConfig,getDefaultMetricsIngestionOutputsConfig,} from '~/ingestion/pipelines/metrics/config'-import { MetricsIngestionConsumer } from '~/ingestion/pipelines/metrics/metrics-ingestion-consumer'import { MetricsPipelineConsumer } from '~/ingestion/pipelines/metrics/metrics-pipeline-consumer'import { createProducerRegistry } from '~/ingestion/pipelines/metrics/outputs/producer-registry'import {@@ -110,10 +109,7 @@ export class IngestionMetricsServer implements NodeServer {const serviceLoaders: (() => Promise<PluginServerService>)[] = []serviceLoaders.push(async () => {- const deps = { teamManager, quotaLimiting, outputs }- const consumer = this.config.METRICS_INGESTION_USE_PIPELINE_FRAMEWORK- ? new MetricsPipelineConsumer(this.config, deps)- : new MetricsIngestionConsumer(this.config, deps)+ const consumer = new MetricsPipelineConsumer(this.config, { teamManager, quotaLimiting, outputs })await consumer.start()return consumer.service})