OpenPanel is an open-source web and product analytics platform, an open-source alternative to Mixpanel with optional self-hosting.
kafka: TLS and SASL auth for external brokers
Carl-Gerhard Lindesvärda8de3833bcba
diff --git a/apps/api/src/config/env.test.ts b/apps/api/src/config/env.test.tsindex 3fea74a6c..7343eadf0 100644--- a/apps/api/src/config/env.test.ts+++ b/apps/api/src/config/env.test.ts@@ -245,6 +245,80 @@ describe('derived values, computed once in the transform', () => {).toBe('own');});+ it('kafka security is plaintext without auth when nothing is set', () => {+ expect(loadConfig(base).core.kafka.security).toEqual({+ ssl: { enabled: false, caPath: undefined, rejectUnauthorized: undefined },+ sasl: undefined,+ });+ });++ it('KAFKA_SSL=true (or 1) enables TLS on its own; the CA path and switch ride along', () => {+ expect(+ loadConfig({ ...base, KAFKA_SSL: 'true' }).core.kafka.security.ssl+ ).toEqual({+ enabled: true,+ caPath: undefined,+ rejectUnauthorized: undefined,+ });+ expect(+ loadConfig({+ ...base,+ KAFKA_SSL: '1',+ KAFKA_SSL_CA_PATH: '/certs/ca.pem',+ KAFKA_SSL_REJECT_UNAUTHORIZED: 'false',+ }).core.kafka.security.ssl+ ).toEqual({+ enabled: true,+ caPath: '/certs/ca.pem',+ rejectUnauthorized: false,+ });+ });++ it('SASL credentials imply TLS and default to scram-sha-512', () => {+ expect(+ loadConfig({+ ...base,+ KAFKA_SASL_USERNAME: 'op',+ KAFKA_SASL_PASSWORD: 'secret',+ }).core.kafka.security+ ).toEqual({+ ssl: { enabled: true, caPath: undefined, rejectUnauthorized: undefined },+ sasl: { mechanism: 'scram-sha-512', username: 'op', password: 'secret' },+ });+ });++ it('accepts every mechanism case-insensitively', () => {+ for (const mechanism of [+ 'plain',+ 'scram-sha-256',+ 'scram-sha-512',+ ] as const) {+ expect(+ loadConfig({+ ...base,+ KAFKA_SASL_USERNAME: 'op',+ KAFKA_SASL_PASSWORD: 'secret',+ KAFKA_SASL_MECHANISM: mechanism.toUpperCase(),+ }).core.kafka.security.sasl?.mechanism+ ).toBe(mechanism);+ }+ });++ it('KAFKA_SSL=false sends SASL over plaintext when asked explicitly', () => {+ expect(+ loadConfig({+ ...base,+ KAFKA_SSL: 'false',+ KAFKA_SASL_USERNAME: 'op',+ KAFKA_SASL_PASSWORD: 'secret',+ KAFKA_SASL_MECHANISM: 'plain',+ }).core.kafka.security+ ).toEqual({+ ssl: { enabled: false, caPath: undefined, rejectUnauthorized: undefined },+ sasl: { mechanism: 'plain', username: 'op', password: 'secret' },+ });+ });+it('the dead-letter list cap defaults to 1000 and takes a positive integer', () => {// The Kafka DLQ topic above is no longer the dead-letter destination// (M20-001) — it is kept only so the seam can be swapped back.@@ -336,6 +410,69 @@ describe('cross-field invariants, checked before the transform', () => {).toThrow(/must exceed the reaper deadman/);});+ it('refuses half a Kafka SASL credential pair, naming the missing half', () => {+ expect(() => loadConfig({ ...base, KAFKA_SASL_USERNAME: 'op' })).toThrow(+ /KAFKA_SASL_PASSWORD is missing/+ );+ expect(() =>+ loadConfig({ ...base, KAFKA_SASL_PASSWORD: 'secret' })+ ).toThrow(/KAFKA_SASL_USERNAME is missing/);+ });++ it('refuses a SASL mechanism without credentials', () => {+ expect(() =>+ loadConfig({ ...base, KAFKA_SASL_MECHANISM: 'plain' })+ ).toThrow(/KAFKA_SASL_MECHANISM is set but/);+ });++ it('refuses an unsupported SASL mechanism before any client exists', () => {+ expect(() =>+ loadConfig({+ ...base,+ KAFKA_SASL_USERNAME: 'op',+ KAFKA_SASL_PASSWORD: 'secret',+ KAFKA_SASL_MECHANISM: 'oauthbearer',+ })+ ).toThrow(+ 'Unsupported KAFKA_SASL_MECHANISM "oauthbearer". Supported: plain, scram-sha-256, scram-sha-512'+ );+ });++ it('refuses TLS-only options on a plaintext connection', () => {+ expect(() =>+ loadConfig({ ...base, KAFKA_SSL_CA_PATH: '/certs/ca.pem' })+ ).toThrow(/require TLS/);+ expect(() =>+ loadConfig({+ ...base,+ KAFKA_SSL: 'false',+ KAFKA_SSL_REJECT_UNAUTHORIZED: 'false',+ })+ ).toThrow(/require TLS/);+ });++ it('refuses a non-boolean KAFKA_SSL', () => {+ expect(() => loadConfig({ ...base, KAFKA_SSL: 'yes' })).toThrow(+ /must be "true" or "false" \(got "yes"\)/+ );+ });++ it('never puts the SASL password in a boot error', () => {+ let message = '';+ try {+ loadConfig({+ ...base,+ KAFKA_SASL_USERNAME: 'op',+ KAFKA_SASL_PASSWORD: 'hunter2-secret',+ KAFKA_SASL_MECHANISM: 'nope',+ });+ } catch (error) {+ message = error instanceof Error ? error.message : String(error);+ }+ expect(message).toContain('Unsupported KAFKA_SASL_MECHANISM');+ expect(message).not.toContain('hunter2-secret');+ });+it('refuses a profile-backfill project list with the flag off', () => {expect(() =>loadConfig({ ...base, EXPERIMENTAL_PROFILE_BACKFILL_PROJECTS: 'p1' })diff --git a/apps/api/src/config/env.ts b/apps/api/src/config/env.tsindex cafbd7fb5..328847e2a 100644--- a/apps/api/src/config/env.ts+++ b/apps/api/src/config/env.ts@@ -27,8 +27,8 @@// documented default on a malformed value, which is what the modules did// when they parsed these themselves.-import type { CoreConfig } from '@openpanel/core';-import { queues } from '@openpanel/core';+import type { CoreConfig, KafkaSaslMechanism } from '@openpanel/core';+import { KAFKA_SASL_MECHANISMS, queues } from '@openpanel/core';import { z } from 'zod';export const ROLE_VALUES = ['api', 'worker', 'all'] as const;@@ -82,6 +82,7 @@ const DEFAULT_KAFKA_HANDLER_MAX_ATTEMPTS = 3;const DEFAULT_INGEST_DEAD_LETTER_MAX_ENTRIES = 1000;const DEFAULT_KAFKA_HANDLER_RETRY_INITIAL_MS = 100;const DEFAULT_KAFKA_HANDLER_RETRY_MAX_MS = 1000;+const DEFAULT_KAFKA_SASL_MECHANISM: KafkaSaslMechanism = 'scram-sha-512';/*** How long a duplicate marker outlives its event (M21-001). It only has to@@ -165,6 +166,54 @@ const trueOrOneSchema = z.preprocess(.transform((value) => value === TRUE_STRING || value === ONE_STRING));+/**+ * `true`/`1` or `false`/`0`, `undefined` when unset. Anything else fails boot:+ * these switch TLS on a broker connection, where a typo must not silently+ * mean "off".+ */+const strictOptionalBoolean = z.preprocess(+ blankToUndefined,+ z+ .string()+ .optional()+ .superRefine((value, ctx) => {+ if (value === undefined || STRICT_BOOLEAN_STRINGS.has(value)) {+ return;+ }+ ctx.addIssue({+ code: z.ZodIssueCode.custom,+ message: `must be "true" or "false" (got "${value}")`,+ });+ })+ .transform((value) =>+ value === undefined+ ? undefined+ : value === TRUE_STRING || value === ONE_STRING+ )+);++/** Case-insensitive; an unknown mechanism fails boot before a client exists. */+const kafkaSaslMechanismSchema = z.preprocess(+ blankToUndefined,+ z+ .string()+ .optional()+ .transform((value) => value?.toLowerCase())+ .superRefine((value, ctx) => {+ if (+ value === undefined ||+ (KAFKA_SASL_MECHANISMS as readonly string[]).includes(value)+ ) {+ return;+ }+ ctx.addIssue({+ code: z.ZodIssueCode.custom,+ message: `Unsupported KAFKA_SASL_MECHANISM "${value}". Supported: ${KAFKA_SASL_MECHANISMS.join(', ')}`,+ });+ })+ .transform((value) => value as KafkaSaslMechanism | undefined)+);+/** Any defined value means on — V1's spelling for DISABLE_WORKERS. */const definedIsTrueSchema = z.preprocess(blankToUndefined,@@ -175,6 +224,12 @@ const definedIsTrueSchema = z.preprocess();const ZERO_STRING = '0';+const STRICT_BOOLEAN_STRINGS = new Set([+ TRUE_STRING,+ ONE_STRING,+ FALSE_STRING,+ ZERO_STRING,+]);/** On unless explicitly switched off with `false` or `0`. */const onUnlessDisabledSchema = z.preprocess(@@ -470,6 +525,13 @@ const rawSchema = z.object({KAFKA_HANDLER_RETRY_MAX_MS: positiveIntWithDefault(DEFAULT_KAFKA_HANDLER_RETRY_MAX_MS),+ /** TLS/SASL for an external broker; unauthenticated plaintext when unset. */+ KAFKA_SSL: strictOptionalBoolean,+ KAFKA_SSL_CA_PATH: optionalString,+ KAFKA_SSL_REJECT_UNAUTHORIZED: strictOptionalBoolean,+ KAFKA_SASL_USERNAME: optionalString,+ KAFKA_SASL_PASSWORD: optionalString,+ KAFKA_SASL_MECHANISM: kafkaSaslMechanismSchema,// --- ingest dead-letter ---INGEST_DEAD_LETTER_MAX_ENTRIES: positiveIntWithDefault(@@ -584,6 +646,53 @@ function checkKafkaHeartbeatFitsSession(});}+/** SASL credentials go over TLS unless KAFKA_SSL=false opts out explicitly. */+function kafkaSslEnabled(raw: RawEnv): boolean {+ const saslEnabled =+ raw.KAFKA_SASL_USERNAME !== undefined ||+ raw.KAFKA_SASL_PASSWORD !== undefined;+ return raw.KAFKA_SSL ?? saslEnabled;+}++/** Half a credential pair, or a mechanism with none, is a misconfiguration. */+function checkKafkaSaslIsComplete(raw: RawEnv, ctx: z.RefinementCtx): void {+ const hasUsername = raw.KAFKA_SASL_USERNAME !== undefined;+ const hasPassword = raw.KAFKA_SASL_PASSWORD !== undefined;+ if (hasUsername && hasPassword) {+ return;+ }+ if (hasUsername || hasPassword) {+ const missing = hasUsername ? 'KAFKA_SASL_PASSWORD' : 'KAFKA_SASL_USERNAME';+ ctx.addIssue({+ code: z.ZodIssueCode.custom,+ message: `Kafka SASL is partially configured: ${missing} is missing (both KAFKA_SASL_USERNAME and KAFKA_SASL_PASSWORD are required)`,+ });+ return;+ }+ if (raw.KAFKA_SASL_MECHANISM !== undefined) {+ ctx.addIssue({+ code: z.ZodIssueCode.custom,+ message:+ 'KAFKA_SASL_MECHANISM is set but KAFKA_SASL_USERNAME and KAFKA_SASL_PASSWORD are missing',+ });+ }+}++/** A CA or a verification switch on a plaintext connection does nothing. */+function checkKafkaTlsOptionsHaveTls(raw: RawEnv, ctx: z.RefinementCtx): void {+ const hasTlsOptions =+ raw.KAFKA_SSL_CA_PATH !== undefined ||+ raw.KAFKA_SSL_REJECT_UNAUTHORIZED !== undefined;+ if (!hasTlsOptions || kafkaSslEnabled(raw)) {+ return;+ }+ ctx.addIssue({+ code: z.ZodIssueCode.custom,+ message:+ 'KAFKA_SSL_CA_PATH / KAFKA_SSL_REJECT_UNAUTHORIZED require TLS; set KAFKA_SSL=true',+ });+}+/** The reaper's deadman closes sessions; below the idle window it closes live ones. */function checkReaperDeadmanOutlivesSession(raw: RawEnv,@@ -666,6 +775,27 @@ function deriveKafkaConfig(raw: RawEnv): CoreConfig['kafka'] {handlerMaxAttempts: raw.KAFKA_HANDLER_MAX_ATTEMPTS,handlerRetryInitialMs: raw.KAFKA_HANDLER_RETRY_INITIAL_MS,handlerRetryMaxMs: raw.KAFKA_HANDLER_RETRY_MAX_MS,+ security: deriveKafkaSecurity(raw),+ };+}++function deriveKafkaSecurity(raw: RawEnv): CoreConfig['kafka']['security'] {+ const hasCredentials =+ raw.KAFKA_SASL_USERNAME !== undefined &&+ raw.KAFKA_SASL_PASSWORD !== undefined;+ return {+ ssl: {+ enabled: kafkaSslEnabled(raw),+ caPath: raw.KAFKA_SSL_CA_PATH,+ rejectUnauthorized: raw.KAFKA_SSL_REJECT_UNAUTHORIZED,+ },+ sasl: hasCredentials+ ? {+ mechanism: raw.KAFKA_SASL_MECHANISM ?? DEFAULT_KAFKA_SASL_MECHANISM,+ username: raw.KAFKA_SASL_USERNAME as string,+ password: raw.KAFKA_SASL_PASSWORD as string,+ }+ : undefined,};}@@ -863,6 +993,8 @@ const envSchema = rawSchemacheckRoleConsumesSomething(raw, ctx);checkOtlpHasKey(raw, ctx);checkKafkaHeartbeatFitsSession(raw, ctx);+ checkKafkaSaslIsComplete(raw, ctx);+ checkKafkaTlsOptionsHaveTls(raw, ctx);checkReaperDeadmanOutlivesSession(raw, ctx);checkVacuumOutlivesReaper(raw, ctx);checkProfileBackfillProjectsHaveAFlag(raw, ctx);diff --git a/docker-compose.yml b/docker-compose.ymlindex 0dda0408b..e891fbbc0 100644--- a/docker-compose.yml+++ b/docker-compose.yml@@ -64,6 +64,25 @@ services:- "127.0.0.1:23092:19092" # Kafka API (external)- "127.0.0.1:23644:9644" # Admin API+ # Opt-in (profile "kafka-auth"): standalone broker that requires SASL, with a+ # TLS listener, for verifying KAFKA_SSL/KAFKA_SASL_* support.+ # See docker/redpanda/auth-entrypoint.sh. Not started by `bun run dock:up`.+ op-rp-auth:+ image: redpandadata/redpanda:v25.1.4+ profiles: ["kafka-auth"]+ entrypoint: ["/auth-entrypoint.sh"]+ volumes:+ - ./docker/data/op-rp-auth-certs:/certs+ - ./docker/redpanda/auth-entrypoint.sh:/auth-entrypoint.sh:ro+ ports:+ - "127.0.0.1:23093:23093" # Kafka API, SASL required, plaintext+ - "127.0.0.1:23094:23094" # Kafka API, SASL required, TLS+ healthcheck:+ test: ["CMD-SHELL", "rpk cluster health -X admin.hosts=localhost:9644 | grep -q 'Healthy:.*true'"]+ interval: 5s+ timeout: 3s+ retries: 20+op-rp-console:image: redpandadata/console:v3.7.2restart: alwaysdiff --git a/docker/redpanda/auth-entrypoint.sh b/docker/redpanda/auth-entrypoint.shnew file mode 100755index 000000000..5987bba6e--- /dev/null+++ b/docker/redpanda/auth-entrypoint.sh@@ -0,0 +1,60 @@+#!/usr/bin/env bash+# Standalone Redpanda that REQUIRES SASL, for verifying the KAFKA_SSL /+# KAFKA_SASL_* support in packages/core/src/modules/ingest against a real+# broker. Not part of the default stack:+#+# bun run dock:kafka-auth # start (compose profile "kafka-auth")+#+# Listeners (both require SASL), bound to 127.0.0.1 by docker-compose.yml:+# localhost:23093 plaintext+# localhost:23094 TLS, self-signed CA written to docker/data/op-rp-auth-certs+# Users: op512 (SCRAM-SHA-512), op256 (SCRAM-SHA-256); PLAIN works with both.+set -euo pipefail++CERT_DIR=/certs+SASL_PORT=23093+TLS_PORT=23094++gen_certs() {+ if [[ -f "$CERT_DIR/ca.crt" && -f "$CERT_DIR/broker.crt" && -f "$CERT_DIR/broker.key" ]]; then+ return+ fi+ echo "generating self-signed TLS certs in $CERT_DIR"+ openssl req -x509 -newkey rsa:2048 -nodes -keyout "$CERT_DIR/ca.key" -out "$CERT_DIR/ca.crt" \+ -days 3650 -subj "/CN=openpanel-dev-ca" 2>/dev/null+ openssl req -newkey rsa:2048 -nodes -keyout "$CERT_DIR/broker.key" -out "$CERT_DIR/broker.csr" \+ -subj "/CN=localhost" 2>/dev/null+ printf "subjectAltName=DNS:localhost,IP:127.0.0.1\n" > "$CERT_DIR/san.ext"+ openssl x509 -req -in "$CERT_DIR/broker.csr" -CA "$CERT_DIR/ca.crt" -CAkey "$CERT_DIR/ca.key" \+ -CAcreateserial -out "$CERT_DIR/broker.crt" -days 3650 -extfile "$CERT_DIR/san.ext" 2>/dev/null+ chmod 644 "$CERT_DIR"/*.crt "$CERT_DIR"/*.key+}++configure_cluster() {+ for _ in $(seq 1 60); do+ if rpk cluster health -X admin.hosts=localhost:9644 >/dev/null 2>&1; then+ break+ fi+ sleep 1+ done+ rpk cluster config set enable_sasl true -X admin.hosts=localhost:9644+ rpk cluster config set sasl_mechanisms '["SCRAM","PLAIN"]' -X admin.hosts=localhost:9644+ rpk cluster config set superusers '["op512","op256"]' -X admin.hosts=localhost:9644+ # Redpanda stores one SCRAM mechanism per user; PLAIN works against either.+ rpk security user create op512 -p op512-secret --mechanism SCRAM-SHA-512 -X admin.hosts=localhost:9644 || true+ rpk security user create op256 -p op256-secret --mechanism SCRAM-SHA-256 -X admin.hosts=localhost:9644 || true+ echo "kafka-auth broker ready"+}++gen_certs+configure_cluster &++exec rpk redpanda start \+ --smp=1 \+ --memory=512M \+ --overprovisioned \+ --mode=dev-container \+ --default-log-level=warn \+ --kafka-addr="sasl://0.0.0.0:$SASL_PORT,tls://0.0.0.0:$TLS_PORT" \+ --advertise-kafka-addr="sasl://localhost:$SASL_PORT,tls://localhost:$TLS_PORT" \+ --set "redpanda.kafka_api_tls=[{\"name\":\"tls\",\"enabled\":true,\"cert_file\":\"$CERT_DIR/broker.crt\",\"key_file\":\"$CERT_DIR/broker.key\",\"truststore_file\":\"$CERT_DIR/ca.crt\",\"require_client_auth\":false}]"diff --git a/packages/core/src/config.ts b/packages/core/src/config.tsindex a61822b40..1ce57f3f3 100644--- a/packages/core/src/config.ts+++ b/packages/core/src/config.ts@@ -85,6 +85,31 @@ export interface PolarConfig {webhookSecret: string | undefined;}+export const KAFKA_SASL_MECHANISMS = [+ 'plain',+ 'scram-sha-256',+ 'scram-sha-512',+] as const;+export type KafkaSaslMechanism = (typeof KAFKA_SASL_MECHANISMS)[number];++/**+ * TLS/SASL for an external broker. Unauthenticated plaintext when nothing is+ * set. The loader has already checked the combination is coherent (a full+ * credential pair, a known mechanism, TLS options only with TLS); the CA file+ * itself is read where the client is built, so an unreadable path fails there+ * with its own message.+ */+export interface KafkaSecurityConfig {+ ssl: {+ enabled: boolean;+ caPath: string | undefined;+ rejectUnauthorized: boolean | undefined;+ };+ sasl:+ | { mechanism: KafkaSaslMechanism; username: string; password: string }+ | undefined;+}+/*** The Kafka transport's knobs. Every one is resolved: kafkajs is handed these* values verbatim, so a `??` at the call site would be a second default.@@ -115,6 +140,7 @@ export interface KafkaConfig {handlerMaxAttempts: number;handlerRetryInitialMs: number;handlerRetryMaxMs: number;+ security: KafkaSecurityConfig;}/** Per-buffer sizing. `undefined` keeps the buffer's own documented default. */diff --git a/packages/core/src/index.ts b/packages/core/src/index.tsindex 459cc168e..fccddad7f 100644--- a/packages/core/src/index.ts+++ b/packages/core/src/index.ts@@ -9,7 +9,14 @@// packages/db reads ClickHouse row JSON through this (P11: packages/json's// only definition already lived here; the barrel is how a package outside// core reaches it, since the exports map has no ./shared/* entry).-export { generateId, generateSecureId, getChartPrevStartEndDate, getSafeJson, resolveDateRange, shortId } from '@openpanel/shared';+export {+ generateId,+ generateSecureId,+ getChartPrevStartEndDate,+ getSafeJson,+ resolveDateRange,+ shortId,+} from '@openpanel/shared';export type {SafeFetchOptions,SafeFetchResult,@@ -23,21 +30,25 @@ export type {// relative import now that it lives inside core (M5-007).export {assertPublicHostname,- assertPublicUrl,assertSafeUrl,+ assertPublicUrl,+ assertSafeUrl,BlockedUrlError,createHash,- createPinnedAgent,createPinnedLookup,+ createPinnedAgent,+ createPinnedLookup,decrypt,decryptCredential,encrypt,encryptCredential,- generateSalt,getDevice,+ generateSalt,+ getDevice,hashPassword,isBlockedIp,- isEncrypted,parseUserAgent,+ isEncrypted,+ parseUserAgent,safeFetch,- safeFetchStream,- verifyPassword+ safeFetchStream,+ verifyPassword,} from '@openpanel/shared/server';// The seven buffers (moved from packages/db/src/buffers, M8-001). Only the// FACTORY is on the barrel: they are boot singletons on `AppDeps`, built once@@ -77,6 +88,10 @@ export {ProviderError,type ProviderErrorOptions,} from './clients/provider-error';+// The one config VALUE the loader needs from core: the accepted SASL+// mechanisms, so `KAFKA_SASL_MECHANISM` is validated against the list the+// client actually supports rather than a second copy of it.+export { KAFKA_SASL_MECHANISMS, type KafkaSaslMechanism } from './config';export type {AppDeps,CoreConfig,diff --git a/packages/core/src/modules/ingest/src/kafka-security.test.ts b/packages/core/src/modules/ingest/src/kafka-security.test.tsnew file mode 100644index 000000000..0bcebf734--- /dev/null+++ b/packages/core/src/modules/ingest/src/kafka-security.test.ts@@ -0,0 +1,117 @@+import { describe, expect, it } from 'bun:test';+import type { KafkaSecurityConfig } from '../../../config';+import { describeKafkaSecurity, resolveKafkaSecurity } from './kafka-security';++const FAKE_PEM = '-----BEGIN CERTIFICATE-----\nfake\n-----END CERTIFICATE-----';++const readFile = (path: string): string => {+ if (path === '/certs/ca.pem') {+ return FAKE_PEM;+ }+ throw new Error(`ENOENT: no such file or directory, open '${path}'`);+};++const plaintext: KafkaSecurityConfig = {+ ssl: { enabled: false, caPath: undefined, rejectUnauthorized: undefined },+ sasl: undefined,+};++const withSasl = (+ mechanism: 'plain' | 'scram-sha-256' | 'scram-sha-512',+ ssl: KafkaSecurityConfig['ssl'] = {+ enabled: true,+ caPath: undefined,+ rejectUnauthorized: undefined,+ }+): KafkaSecurityConfig => ({+ ssl,+ sasl: { mechanism, username: 'op', password: 'secret' },+});++describe('resolveKafkaSecurity', () => {+ it('is plaintext without auth when nothing is configured', () => {+ expect(resolveKafkaSecurity(plaintext, readFile)).toEqual({+ ssl: false,+ sasl: undefined,+ });+ });++ it('is a bare `true` for TLS with the system trust store', () => {+ expect(+ resolveKafkaSecurity(+ { ...plaintext, ssl: { ...plaintext.ssl, enabled: true } },+ readFile+ )+ ).toEqual({ ssl: true, sasl: undefined });+ });++ it('passes a custom CA and rejectUnauthorized through to tls options', () => {+ expect(+ resolveKafkaSecurity(+ {+ ...plaintext,+ ssl: {+ enabled: true,+ caPath: '/certs/ca.pem',+ rejectUnauthorized: false,+ },+ },+ readFile+ ).ssl+ ).toEqual({ ca: [FAKE_PEM], rejectUnauthorized: false });+ });++ it('fails with the path when the CA file cannot be read', () => {+ expect(() =>+ resolveKafkaSecurity(+ {+ ...plaintext,+ ssl: {+ enabled: true,+ caPath: '/missing.pem',+ rejectUnauthorized: undefined,+ },+ },+ readFile+ )+ ).toThrow(/Failed to read KAFKA_SSL_CA_PATH "\/missing.pem"/);+ });++ it('hands SASL credentials to kafkajs unchanged, per mechanism', () => {+ for (const mechanism of [+ 'plain',+ 'scram-sha-256',+ 'scram-sha-512',+ ] as const) {+ expect(resolveKafkaSecurity(withSasl(mechanism), readFile)).toEqual({+ ssl: true,+ sasl: { mechanism, username: 'op', password: 'secret' },+ });+ }+ });++ it('allows SASL over plaintext when the loader resolved TLS off', () => {+ expect(+ resolveKafkaSecurity(withSasl('plain', plaintext.ssl), readFile)+ ).toEqual({+ ssl: false,+ sasl: { mechanism: 'plain', username: 'op', password: 'secret' },+ });+ });+});++describe('describeKafkaSecurity', () => {+ it('summarises without credential values', () => {+ const resolved = resolveKafkaSecurity(withSasl('scram-sha-512'), readFile);+ expect(describeKafkaSecurity(resolved)).toEqual({+ ssl: true,+ sasl: 'scram-sha-512',+ });+ expect(JSON.stringify(describeKafkaSecurity(resolved))).not.toContain(+ 'secret'+ );+ expect(+ describeKafkaSecurity(resolveKafkaSecurity(plaintext, readFile))+ ).toEqual({ ssl: false, sasl: null });+ });+});diff --git a/packages/core/src/modules/ingest/src/kafka-security.ts b/packages/core/src/modules/ingest/src/kafka-security.tsnew file mode 100644index 000000000..d3ddab5be--- /dev/null+++ b/packages/core/src/modules/ingest/src/kafka-security.ts@@ -0,0 +1,62 @@+// The kafkajs `ssl` / `sasl` options for an external broker, built from the+// already-validated `KafkaSecurityConfig`. The only thing that can still fail+// here is reading the CA file, and it fails with the path in the message.++import { readFileSync } from 'node:fs';+import type { ConnectionOptions } from 'node:tls';+import type { SASLOptions } from 'kafkajs';+import type { KafkaSaslMechanism, KafkaSecurityConfig } from '../../../config';++export interface ResolvedKafkaSecurity {+ ssl: ConnectionOptions | boolean;+ sasl: SASLOptions | undefined;+}++export type ReadPemFile = (path: string) => string;++const readPemFile: ReadPemFile = (path) => readFileSync(path, 'utf8');++const resolveSsl = (+ ssl: KafkaSecurityConfig['ssl'],+ readFile: ReadPemFile+): ConnectionOptions | boolean => {+ if (!ssl.enabled) {+ return false;+ }+ const options: ConnectionOptions = {};+ if (ssl.caPath) {+ try {+ options.ca = [readFile(ssl.caPath)];+ } catch (err) {+ throw new Error(+ `Failed to read KAFKA_SSL_CA_PATH "${ssl.caPath}": ${err instanceof Error ? err.message : String(err)}`+ );+ }+ }+ if (ssl.rejectUnauthorized !== undefined) {+ options.rejectUnauthorized = ssl.rejectUnauthorized;+ }+ return Object.keys(options).length > 0 ? options : true;+};++export const resolveKafkaSecurity = (+ security: KafkaSecurityConfig,+ readFile: ReadPemFile = readPemFile+): ResolvedKafkaSecurity => ({+ ssl: resolveSsl(security.ssl, readFile),+ sasl: security.sasl+ ? {+ mechanism: security.sasl.mechanism,+ username: security.sasl.username,+ password: security.sasl.password,+ }+ : undefined,+});++/** Log-safe summary: never includes credential values. */+export const describeKafkaSecurity = (+ security: ResolvedKafkaSecurity+): { ssl: boolean; sasl: KafkaSaslMechanism | null } => ({+ ssl: security.ssl !== false,+ sasl: security.sasl ? (security.sasl.mechanism as KafkaSaslMechanism) : null,+});diff --git a/packages/core/src/modules/ingest/src/kafka.ts b/packages/core/src/modules/ingest/src/kafka.tsindex 02c75f6f1..437224ce0 100644--- a/packages/core/src/modules/ingest/src/kafka.ts+++ b/packages/core/src/modules/ingest/src/kafka.ts@@ -25,6 +25,11 @@ import { createLogger, type ILogger } from '../../../pino-logger';// message shape it hands to this producer, and both files are now siblings.import type { DeadLetterMessage } from './consumer';import type { IncomingEventPayload } from './incoming-event';+import {+ describeKafkaSecurity,+ type ResolvedKafkaSecurity,+ resolveKafkaSecurity,+} from './kafka-security';import {createProducerBatcher,type ProducerBatcher,@@ -59,18 +64,31 @@ export const assertKafkaConfigured = (config: KafkaConfig): void => {'KAFKA_BROKERS is not set. Kafka/Redpanda is the only events transport — set KAFKA_BROKERS to a comma-separated broker list.');}+ // Reads the CA file, so an unreadable KAFKA_SSL_CA_PATH fails here at boot+ // rather than on the first produce/consume.+ getKafkaSecurity(config);+};++// Memoised like the client below: the CA file is read once per process.+let kafkaSecurity: ResolvedKafkaSecurity | null = null;+const getKafkaSecurity = (config: KafkaConfig): ResolvedKafkaSecurity => {+ kafkaSecurity ??= resolveKafkaSecurity(config.security);+ return kafkaSecurity;};let kafka: Kafka | null = null;const getKafka = (config: CoreConfig): Kafka => {assertKafkaConfigured(config.kafka);if (!kafka) {+ const security = getKafkaSecurity(config.kafka);kafka = new Kafka({clientId: config.kafka.clientId,brokers: config.kafka.brokers,logLevel: logLevel.WARN,requestTimeout: config.kafka.requestTimeoutMs,connectionTimeout: config.kafka.connectionTimeoutMs,+ ssl: security.ssl,+ sasl: security.sasl,});}return kafka;@@ -112,6 +130,8 @@ const getProducer = async (config: CoreConfig): Promise<Producer> => {{brokers: config.kafka.brokers,topic: config.kafka.eventsTopic,+ // `ssl: bool` and the mechanism only — never a credential.+ ...describeKafkaSecurity(getKafkaSecurity(config.kafka)),// The throughput knobs, logged once, so a measurement run can be// tied to the configuration that produced it.maxInFlight: tuning.maxInFlight,diff --git a/packages/core/src/modules/ingest/src/producer-tuning.test.ts b/packages/core/src/modules/ingest/src/producer-tuning.test.tsindex c1f305a0b..f60d70bfb 100644--- a/packages/core/src/modules/ingest/src/producer-tuning.test.ts+++ b/packages/core/src/modules/ingest/src/producer-tuning.test.ts@@ -38,6 +38,10 @@ const kafkaConfig = (tuning: Partial<KafkaConfig> = {}): KafkaConfig =>handlerMaxAttempts: 3,handlerRetryInitialMs: 100,handlerRetryMaxMs: 1000,+ security: {+ ssl: { enabled: false, caPath: undefined, rejectUnauthorized: undefined },+ sasl: undefined,+ },...tuning,}) satisfies KafkaConfig;diff --git a/packages/core/test/config-fixture.ts b/packages/core/test/config-fixture.tsindex 5a7fa0307..8f30614a6 100644--- a/packages/core/test/config-fixture.ts+++ b/packages/core/test/config-fixture.ts@@ -81,6 +81,14 @@ export function testCoreConfig(handlerMaxAttempts: 3,handlerRetryInitialMs: 100,handlerRetryMaxMs: 1000,+ security: {+ ssl: {+ enabled: false,+ caPath: undefined,+ rejectUnauthorized: undefined,+ },+ sasl: undefined,+ },},buffers: {asyncInserts: false,