:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(ingestion): add a blackhole producer slot for single outputs (#120049)
José Sequeira15427b848b79
diff --git a/nodejs/src/common/outputs/README.md b/nodejs/src/common/outputs/README.mdindex 60291c8ddb2e..6095bb8b9b57 100644--- a/nodejs/src/common/outputs/README.md+++ b/nodejs/src/common/outputs/README.md@@ -24,7 +24,7 @@ Previously, pipelines received raw `KafkaProducerWrapper` instances and hardcode## Concepts-A **producer** is a Kafka connection configured via the server config object. Each producer has a name (e.g. `'INGESTION_DOWNSTREAM'`) and a mapping from config key names to rdkafka config keys. `KafkaProducerRegistryBuilder` creates all producers at startup, returning a typed `KafkaProducerRegistry<P>` where `P` is the union of registered producer names. The producer name is set on the `KafkaProducerWrapper` instance and used in metrics labels.+A **producer** is a Kafka connection configured via the server config object. Each producer has a name (e.g. `'INGESTION_DOWNSTREAM'`) and a mapping from config key names to rdkafka config keys. `KafkaProducerRegistryBuilder` creates all producers at startup, returning a typed `KafkaProducerRegistry<P>` where `P` is the union of registered producer names. The producer name is set on the `KafkaProducerWrapper` instance and used in metrics labels. A producer registered with `registerBlackhole` connects to no broker and discards every message, counted in `kafka_producer_messages_discarded_total`; the ingestion registry exposes one as `INGESTION_BLACKHOLE`, so a chart can route a single output nowhere while the other slots keep producing.A **target** is a single Kafka destination: a topic, a producer, and the producer's name. An output contains one or more targets.diff --git a/nodejs/src/common/outputs/kafka-producer-registry-builder.test.ts b/nodejs/src/common/outputs/kafka-producer-registry-builder.test.tsindex 96e398e094b8..fe41785973b1 100644--- a/nodejs/src/common/outputs/kafka-producer-registry-builder.test.ts+++ b/nodejs/src/common/outputs/kafka-producer-registry-builder.test.ts@@ -116,6 +116,23 @@ describe('KafkaProducerRegistryBuilder', () => {])})+ it('registerBlackhole adds a slot that connects to no broker and discards while the others connect', async () => {+ kafkaProducerMessagesDiscardedCounter.reset()+ const config = { BROKER_LIST: 'kafka:9092', LINGER_MS: '20' }++ const registry = await new KafkaProducerRegistryBuilder(undefined)+ .register('DEFAULT', configMap)+ .registerBlackhole('VOID')+ .build(config)+ await registry.getProducer('VOID').produce({ topic: 'events', key: null, value: Buffer.from('v') })++ expect(KafkaProducerWrapper.createWithConfig).toHaveBeenCalledTimes(1)+ expect(KafkaProducerWrapper.createWithConfig).toHaveBeenCalledWith(undefined, expect.any(Object), 'DEFAULT')+ expect((await kafkaProducerMessagesDiscardedCounter.get()).values).toEqual([+ expect.objectContaining({ labels: { topic_name: 'events', producer_name: 'VOID' }, value: 1 }),+ ])+ })+it('builds an empty registry when no producers are registered', async () => {const registry = await new KafkaProducerRegistryBuilder(undefined).build({})diff --git a/nodejs/src/common/outputs/kafka-producer-registry-builder.ts b/nodejs/src/common/outputs/kafka-producer-registry-builder.tsindex 593db1627db7..1f160f3eafa7 100644--- a/nodejs/src/common/outputs/kafka-producer-registry-builder.ts+++ b/nodejs/src/common/outputs/kafka-producer-registry-builder.ts@@ -36,6 +36,7 @@ function redactConfig(config: ProducerGlobalConfig): Record<string, unknown> {*/export class KafkaProducerRegistryBuilder<P extends string = never, CK extends string = never> {private registrations = new Map<string, Partial<Record<AllowedConfigKey, CK>>>()+ private blackholes = new Set<string>()constructor(private kafkaClientRack: string | undefined) {}@@ -51,10 +52,19 @@ export class KafkaProducerRegistryBuilder<P extends string = never, CK extends s): KafkaProducerRegistryBuilder<P | Name, CK | ConfigKeys> {const next = new KafkaProducerRegistryBuilder<P | Name, CK | ConfigKeys>(this.kafkaClientRack)next.registrations = new Map(this.registrations)+ next.blackholes = new Set(this.blackholes)next.registrations.set(name, configMap)return next}+ registerBlackhole<Name extends string>(name: Name): KafkaProducerRegistryBuilder<P | Name, CK> {+ const next = new KafkaProducerRegistryBuilder<P | Name, CK>(this.kafkaClientRack)+ next.registrations = new Map(this.registrations)+ next.blackholes = new Set(this.blackholes)+ next.blackholes.add(name)+ return next+ }+/*** Create all registered producers and return an immutable registry.*@@ -62,7 +72,10 @@ export class KafkaProducerRegistryBuilder<P extends string = never, CK extends s* Connects to brokers in parallel. Throws if any producer fails to connect.*/async build(config: Record<CK, string>): Promise<KafkaProducerRegistry<P>> {- const producers: Record<string, KafkaProducerWrapper> = {}+ const producers: Record<string, KafkaProducer> = {}+ for (const name of this.blackholes) {+ producers[name] = new BlackholeKafkaProducer(name)+ }await Promise.all(Array.from(this.registrations.entries()).map(async ([name, configMap]) => {@@ -87,14 +100,14 @@ export class KafkaProducerRegistryBuilder<P extends string = never, CK extends s)// TypeScript cannot verify that an imperatively-built Record has all keys of a- // generic union P. The builder guarantees this: every `register()` call adds an- // entry to `this.registrations`, and `build()` creates a producer for each entry.- return new KafkaProducerRegistry<P>(producers as Record<P, KafkaProducerWrapper>)+ // generic union P. The builder guarantees this: every `register()` and+ // `registerBlackhole()` call adds a name, and `build()` creates a producer for each.+ return new KafkaProducerRegistry<P>(producers as Record<P, KafkaProducer>)}buildBlackhole(): KafkaProducerRegistry<P> {const producers: Record<string, KafkaProducer> = {}- for (const name of this.registrations.keys()) {+ for (const name of [...this.registrations.keys(), ...this.blackholes]) {logger.warn('⚠️', `Producer "${name}" is a blackhole and discards every message`)producers[name] = new BlackholeKafkaProducer(name)}diff --git a/nodejs/src/ingestion/common/outputs/producer-registry.test.ts b/nodejs/src/ingestion/common/outputs/producer-registry.test.tsindex 0653ba57f225..c2f25776e8ad 100644--- a/nodejs/src/ingestion/common/outputs/producer-registry.test.ts+++ b/nodejs/src/ingestion/common/outputs/producer-registry.test.ts@@ -1,7 +1,12 @@+import { BlackholeKafkaProducer } from '~/common/kafka/blackhole-producer'import { KafkaProducerWrapper } from '~/common/kafka/producer'import { buildIngestionProducerRegistry } from './producer-registry'-import { getDefaultKafkaDownstreamProducerEnvConfig, getDefaultKafkaUpstreamProducerEnvConfig } from './producers'+import {+ INGESTION_BLACKHOLE_PRODUCER,+ getDefaultKafkaDownstreamProducerEnvConfig,+ getDefaultKafkaUpstreamProducerEnvConfig,+} from './producers'jest.mock('~/common/kafka/producer')@@ -17,8 +22,12 @@ describe('buildIngestionProducerRegistry', () => {[true, 0],[false, 2],])('with INGESTION_OUTPUTS_DISABLED=%s connects %i Kafka producers', async (disabled, connected) => {- await buildIngestionProducerRegistry(undefined, { ...config, INGESTION_OUTPUTS_DISABLED: disabled })+ const registry = await buildIngestionProducerRegistry(undefined, {+ ...config,+ INGESTION_OUTPUTS_DISABLED: disabled,+ })expect(KafkaProducerWrapper.createWithConfig).toHaveBeenCalledTimes(connected)+ expect(registry.getProducer(INGESTION_BLACKHOLE_PRODUCER)).toBeInstanceOf(BlackholeKafkaProducer)})})diff --git a/nodejs/src/ingestion/common/outputs/producer-registry.ts b/nodejs/src/ingestion/common/outputs/producer-registry.tsindex c683023f0be5..42faf8438775 100644--- a/nodejs/src/ingestion/common/outputs/producer-registry.ts+++ b/nodejs/src/ingestion/common/outputs/producer-registry.ts@@ -2,6 +2,7 @@ import { KafkaProducerRegistry } from '~/common/outputs/kafka-producer-registry'import { KafkaProducerRegistryBuilder } from '~/common/outputs/kafka-producer-registry-builder'import {+ INGESTION_BLACKHOLE_PRODUCER,INGESTION_DOWNSTREAM_PRODUCER,INGESTION_DOWNSTREAM_PRODUCER_CONFIG_MAP,INGESTION_UPSTREAM_PRODUCER,@@ -14,7 +15,7 @@ import {* (general, error-tracking, ingestion-api). Each of these must wire both slots' brokers in* charts — an unconfigured slot falls through to the schema-default broker (`kafka:9092`),* which is reachable in dev/hobby but not in prod, so a missing wire fails fast there.- * Session replay composes its own set instead.+ * The BLACKHOLE slot needs no wiring. Session replay composes its own set instead.** Lives in its own module (not producers.ts) so that importing the producer slot* names/maps — which config files do at module-eval time — does not transitively pull in@@ -24,6 +25,7 @@ function createIngestionProducerRegistry(kafkaClientRack: string | undefined) {return new KafkaProducerRegistryBuilder(kafkaClientRack).register(INGESTION_UPSTREAM_PRODUCER, INGESTION_UPSTREAM_PRODUCER_CONFIG_MAP).register(INGESTION_DOWNSTREAM_PRODUCER, INGESTION_DOWNSTREAM_PRODUCER_CONFIG_MAP)+ .registerBlackhole(INGESTION_BLACKHOLE_PRODUCER)}type ProducerRegistryConfig = Parameters<ReturnType<typeof createIngestionProducerRegistry>['build']>[0] & {diff --git a/nodejs/src/ingestion/common/outputs/producers.ts b/nodejs/src/ingestion/common/outputs/producers.tsindex a07e753de8ba..a9421b27e461 100644--- a/nodejs/src/ingestion/common/outputs/producers.ts+++ b/nodejs/src/ingestion/common/outputs/producers.ts@@ -11,6 +11,7 @@ import type { AllowedConfigKey } from '~/common/outputs/kafka-producer-config'// Cluster-accurate slots:// INGESTION_UPSTREAM — dedicated ingestion cluster; re-consumed topics (overflow/async/dlq)// INGESTION_DOWNSTREAM — warpstream-ingestion cluster; ClickHouse-bound outputs+// INGESTION_BLACKHOLE — no cluster; discards and counts every message routed to it// Pipeline-specific slots (e.g. session replay's warpstream-replay producer) live in their own module./** UPSTREAM — dedicated ingestion cluster; re-consumed topics (overflow/async/dlq). */@@ -21,8 +22,12 @@ export type IngestionUpstreamProducer = typeof INGESTION_UPSTREAM_PRODUCERexport const INGESTION_DOWNSTREAM_PRODUCER = 'INGESTION_DOWNSTREAM' as constexport type IngestionDownstreamProducer = typeof INGESTION_DOWNSTREAM_PRODUCER+/** BLACKHOLE — no cluster; an output routed here is discarded and counted. */+export const INGESTION_BLACKHOLE_PRODUCER = 'INGESTION_BLACKHOLE' as const+export type IngestionBlackholeProducer = typeof INGESTION_BLACKHOLE_PRODUCER+/** Union of all known producer names. Extend this as new producers are added. */-export type ProducerName = IngestionUpstreamProducer | IngestionDownstreamProducer+export type ProducerName = IngestionUpstreamProducer | IngestionDownstreamProducer | IngestionBlackholeProducer// =============================================================================// Cluster-accurate producer slots: env-var maps and their default env config.