Skip to content

OpenPanel is an open-source web and product analytics platform, an open-source alternative to Mixpanel with optional self-hosting.

kafka: TLS and SASL auth for external brokers

Carl-Gerhard Lindesvärda8de3833bcba
diff --git a/apps/api/src/config/env.test.ts b/apps/api/src/config/env.test.ts
index 3fea74a6c..7343eadf0 100644
--- a/apps/api/src/config/env.test.ts
+++ b/apps/api/src/config/env.test.ts
@@ -245,6 +245,80 @@ describe('derived values, computed once in the transform', () => {
).toBe('own');
});
+ it('kafka security is plaintext without auth when nothing is set', () => {
+ expect(loadConfig(base).core.kafka.security).toEqual({
+ ssl: { enabled: false, caPath: undefined, rejectUnauthorized: undefined },
+ sasl: undefined,
+ });
+ });
+
+ it('KAFKA_SSL=true (or 1) enables TLS on its own; the CA path and switch ride along', () => {
+ expect(
+ loadConfig({ ...base, KAFKA_SSL: 'true' }).core.kafka.security.ssl
+ ).toEqual({
+ enabled: true,
+ caPath: undefined,
+ rejectUnauthorized: undefined,
+ });
+ expect(
+ loadConfig({
+ ...base,
+ KAFKA_SSL: '1',
+ KAFKA_SSL_CA_PATH: '/certs/ca.pem',
+ KAFKA_SSL_REJECT_UNAUTHORIZED: 'false',
+ }).core.kafka.security.ssl
+ ).toEqual({
+ enabled: true,
+ caPath: '/certs/ca.pem',
+ rejectUnauthorized: false,
+ });
+ });
+
+ it('SASL credentials imply TLS and default to scram-sha-512', () => {
+ expect(
+ loadConfig({
+ ...base,
+ KAFKA_SASL_USERNAME: 'op',
+ KAFKA_SASL_PASSWORD: 'secret',
+ }).core.kafka.security
+ ).toEqual({
+ ssl: { enabled: true, caPath: undefined, rejectUnauthorized: undefined },
+ sasl: { mechanism: 'scram-sha-512', username: 'op', password: 'secret' },
+ });
+ });
+
+ it('accepts every mechanism case-insensitively', () => {
+ for (const mechanism of [
+ 'plain',
+ 'scram-sha-256',
+ 'scram-sha-512',
+ ] as const) {
+ expect(
+ loadConfig({
+ ...base,
+ KAFKA_SASL_USERNAME: 'op',
+ KAFKA_SASL_PASSWORD: 'secret',
+ KAFKA_SASL_MECHANISM: mechanism.toUpperCase(),
+ }).core.kafka.security.sasl?.mechanism
+ ).toBe(mechanism);
+ }
+ });
+
+ it('KAFKA_SSL=false sends SASL over plaintext when asked explicitly', () => {
+ expect(
+ loadConfig({
+ ...base,
+ KAFKA_SSL: 'false',
+ KAFKA_SASL_USERNAME: 'op',
+ KAFKA_SASL_PASSWORD: 'secret',
+ KAFKA_SASL_MECHANISM: 'plain',
+ }).core.kafka.security
+ ).toEqual({
+ ssl: { enabled: false, caPath: undefined, rejectUnauthorized: undefined },
+ sasl: { mechanism: 'plain', username: 'op', password: 'secret' },
+ });
+ });
+
it('the dead-letter list cap defaults to 1000 and takes a positive integer', () => {
// The Kafka DLQ topic above is no longer the dead-letter destination
// (M20-001) — it is kept only so the seam can be swapped back.
@@ -336,6 +410,69 @@ describe('cross-field invariants, checked before the transform', () => {
).toThrow(/must exceed the reaper deadman/);
});
+ it('refuses half a Kafka SASL credential pair, naming the missing half', () => {
+ expect(() => loadConfig({ ...base, KAFKA_SASL_USERNAME: 'op' })).toThrow(
+ /KAFKA_SASL_PASSWORD is missing/
+ );
+ expect(() =>
+ loadConfig({ ...base, KAFKA_SASL_PASSWORD: 'secret' })
+ ).toThrow(/KAFKA_SASL_USERNAME is missing/);
+ });
+
+ it('refuses a SASL mechanism without credentials', () => {
+ expect(() =>
+ loadConfig({ ...base, KAFKA_SASL_MECHANISM: 'plain' })
+ ).toThrow(/KAFKA_SASL_MECHANISM is set but/);
+ });
+
+ it('refuses an unsupported SASL mechanism before any client exists', () => {
+ expect(() =>
+ loadConfig({
+ ...base,
+ KAFKA_SASL_USERNAME: 'op',
+ KAFKA_SASL_PASSWORD: 'secret',
+ KAFKA_SASL_MECHANISM: 'oauthbearer',
+ })
+ ).toThrow(
+ 'Unsupported KAFKA_SASL_MECHANISM "oauthbearer". Supported: plain, scram-sha-256, scram-sha-512'
+ );
+ });
+
+ it('refuses TLS-only options on a plaintext connection', () => {
+ expect(() =>
+ loadConfig({ ...base, KAFKA_SSL_CA_PATH: '/certs/ca.pem' })
+ ).toThrow(/require TLS/);
+ expect(() =>
+ loadConfig({
+ ...base,
+ KAFKA_SSL: 'false',
+ KAFKA_SSL_REJECT_UNAUTHORIZED: 'false',
+ })
+ ).toThrow(/require TLS/);
+ });
+
+ it('refuses a non-boolean KAFKA_SSL', () => {
+ expect(() => loadConfig({ ...base, KAFKA_SSL: 'yes' })).toThrow(
+ /must be "true" or "false" \(got "yes"\)/
+ );
+ });
+
+ it('never puts the SASL password in a boot error', () => {
+ let message = '';
+ try {
+ loadConfig({
+ ...base,
+ KAFKA_SASL_USERNAME: 'op',
+ KAFKA_SASL_PASSWORD: 'hunter2-secret',
+ KAFKA_SASL_MECHANISM: 'nope',
+ });
+ } catch (error) {
+ message = error instanceof Error ? error.message : String(error);
+ }
+ expect(message).toContain('Unsupported KAFKA_SASL_MECHANISM');
+ expect(message).not.toContain('hunter2-secret');
+ });
+
it('refuses a profile-backfill project list with the flag off', () => {
expect(() =>
loadConfig({ ...base, EXPERIMENTAL_PROFILE_BACKFILL_PROJECTS: 'p1' })
diff --git a/apps/api/src/config/env.ts b/apps/api/src/config/env.ts
index cafbd7fb5..328847e2a 100644
--- a/apps/api/src/config/env.ts
+++ b/apps/api/src/config/env.ts
@@ -27,8 +27,8 @@
// documented default on a malformed value, which is what the modules did
// when they parsed these themselves.
-import type { CoreConfig } from '@openpanel/core';
-import { queues } from '@openpanel/core';
+import type { CoreConfig, KafkaSaslMechanism } from '@openpanel/core';
+import { KAFKA_SASL_MECHANISMS, queues } from '@openpanel/core';
import { z } from 'zod';
export const ROLE_VALUES = ['api', 'worker', 'all'] as const;
@@ -82,6 +82,7 @@ const DEFAULT_KAFKA_HANDLER_MAX_ATTEMPTS = 3;
const DEFAULT_INGEST_DEAD_LETTER_MAX_ENTRIES = 1000;
const DEFAULT_KAFKA_HANDLER_RETRY_INITIAL_MS = 100;
const DEFAULT_KAFKA_HANDLER_RETRY_MAX_MS = 1000;
+const DEFAULT_KAFKA_SASL_MECHANISM: KafkaSaslMechanism = 'scram-sha-512';
/**
* How long a duplicate marker outlives its event (M21-001). It only has to
@@ -165,6 +166,54 @@ const trueOrOneSchema = z.preprocess(
.transform((value) => value === TRUE_STRING || value === ONE_STRING)
);
+/**
+ * `true`/`1` or `false`/`0`, `undefined` when unset. Anything else fails boot:
+ * these switch TLS on a broker connection, where a typo must not silently
+ * mean "off".
+ */
+const strictOptionalBoolean = z.preprocess(
+ blankToUndefined,
+ z
+ .string()
+ .optional()
+ .superRefine((value, ctx) => {
+ if (value === undefined || STRICT_BOOLEAN_STRINGS.has(value)) {
+ return;
+ }
+ ctx.addIssue({
+ code: z.ZodIssueCode.custom,
+ message: `must be "true" or "false" (got "${value}")`,
+ });
+ })
+ .transform((value) =>
+ value === undefined
+ ? undefined
+ : value === TRUE_STRING || value === ONE_STRING
+ )
+);
+
+/** Case-insensitive; an unknown mechanism fails boot before a client exists. */
+const kafkaSaslMechanismSchema = z.preprocess(
+ blankToUndefined,
+ z
+ .string()
+ .optional()
+ .transform((value) => value?.toLowerCase())
+ .superRefine((value, ctx) => {
+ if (
+ value === undefined ||
+ (KAFKA_SASL_MECHANISMS as readonly string[]).includes(value)
+ ) {
+ return;
+ }
+ ctx.addIssue({
+ code: z.ZodIssueCode.custom,
+ message: `Unsupported KAFKA_SASL_MECHANISM "${value}". Supported: ${KAFKA_SASL_MECHANISMS.join(', ')}`,
+ });
+ })
+ .transform((value) => value as KafkaSaslMechanism | undefined)
+);
+
/** Any defined value means on — V1's spelling for DISABLE_WORKERS. */
const definedIsTrueSchema = z.preprocess(
blankToUndefined,
@@ -175,6 +224,12 @@ const definedIsTrueSchema = z.preprocess(
);
const ZERO_STRING = '0';
+const STRICT_BOOLEAN_STRINGS = new Set([
+ TRUE_STRING,
+ ONE_STRING,
+ FALSE_STRING,
+ ZERO_STRING,
+]);
/** On unless explicitly switched off with `false` or `0`. */
const onUnlessDisabledSchema = z.preprocess(
@@ -470,6 +525,13 @@ const rawSchema = z.object({
KAFKA_HANDLER_RETRY_MAX_MS: positiveIntWithDefault(
DEFAULT_KAFKA_HANDLER_RETRY_MAX_MS
),
+ /** TLS/SASL for an external broker; unauthenticated plaintext when unset. */
+ KAFKA_SSL: strictOptionalBoolean,
+ KAFKA_SSL_CA_PATH: optionalString,
+ KAFKA_SSL_REJECT_UNAUTHORIZED: strictOptionalBoolean,
+ KAFKA_SASL_USERNAME: optionalString,
+ KAFKA_SASL_PASSWORD: optionalString,
+ KAFKA_SASL_MECHANISM: kafkaSaslMechanismSchema,
// --- ingest dead-letter ---
INGEST_DEAD_LETTER_MAX_ENTRIES: positiveIntWithDefault(
@@ -584,6 +646,53 @@ function checkKafkaHeartbeatFitsSession(
});
}
+/** SASL credentials go over TLS unless KAFKA_SSL=false opts out explicitly. */
+function kafkaSslEnabled(raw: RawEnv): boolean {
+ const saslEnabled =
+ raw.KAFKA_SASL_USERNAME !== undefined ||
+ raw.KAFKA_SASL_PASSWORD !== undefined;
+ return raw.KAFKA_SSL ?? saslEnabled;
+}
+
+/** Half a credential pair, or a mechanism with none, is a misconfiguration. */
+function checkKafkaSaslIsComplete(raw: RawEnv, ctx: z.RefinementCtx): void {
+ const hasUsername = raw.KAFKA_SASL_USERNAME !== undefined;
+ const hasPassword = raw.KAFKA_SASL_PASSWORD !== undefined;
+ if (hasUsername && hasPassword) {
+ return;
+ }
+ if (hasUsername || hasPassword) {
+ const missing = hasUsername ? 'KAFKA_SASL_PASSWORD' : 'KAFKA_SASL_USERNAME';
+ ctx.addIssue({
+ code: z.ZodIssueCode.custom,
+ message: `Kafka SASL is partially configured: ${missing} is missing (both KAFKA_SASL_USERNAME and KAFKA_SASL_PASSWORD are required)`,
+ });
+ return;
+ }
+ if (raw.KAFKA_SASL_MECHANISM !== undefined) {
+ ctx.addIssue({
+ code: z.ZodIssueCode.custom,
+ message:
+ 'KAFKA_SASL_MECHANISM is set but KAFKA_SASL_USERNAME and KAFKA_SASL_PASSWORD are missing',
+ });
+ }
+}
+
+/** A CA or a verification switch on a plaintext connection does nothing. */
+function checkKafkaTlsOptionsHaveTls(raw: RawEnv, ctx: z.RefinementCtx): void {
+ const hasTlsOptions =
+ raw.KAFKA_SSL_CA_PATH !== undefined ||
+ raw.KAFKA_SSL_REJECT_UNAUTHORIZED !== undefined;
+ if (!hasTlsOptions || kafkaSslEnabled(raw)) {
+ return;
+ }
+ ctx.addIssue({
+ code: z.ZodIssueCode.custom,
+ message:
+ 'KAFKA_SSL_CA_PATH / KAFKA_SSL_REJECT_UNAUTHORIZED require TLS; set KAFKA_SSL=true',
+ });
+}
+
/** The reaper's deadman closes sessions; below the idle window it closes live ones. */
function checkReaperDeadmanOutlivesSession(
raw: RawEnv,
@@ -666,6 +775,27 @@ function deriveKafkaConfig(raw: RawEnv): CoreConfig['kafka'] {
handlerMaxAttempts: raw.KAFKA_HANDLER_MAX_ATTEMPTS,
handlerRetryInitialMs: raw.KAFKA_HANDLER_RETRY_INITIAL_MS,
handlerRetryMaxMs: raw.KAFKA_HANDLER_RETRY_MAX_MS,
+ security: deriveKafkaSecurity(raw),
+ };
+}
+
+function deriveKafkaSecurity(raw: RawEnv): CoreConfig['kafka']['security'] {
+ const hasCredentials =
+ raw.KAFKA_SASL_USERNAME !== undefined &&
+ raw.KAFKA_SASL_PASSWORD !== undefined;
+ return {
+ ssl: {
+ enabled: kafkaSslEnabled(raw),
+ caPath: raw.KAFKA_SSL_CA_PATH,
+ rejectUnauthorized: raw.KAFKA_SSL_REJECT_UNAUTHORIZED,
+ },
+ sasl: hasCredentials
+ ? {
+ mechanism: raw.KAFKA_SASL_MECHANISM ?? DEFAULT_KAFKA_SASL_MECHANISM,
+ username: raw.KAFKA_SASL_USERNAME as string,
+ password: raw.KAFKA_SASL_PASSWORD as string,
+ }
+ : undefined,
};
}
@@ -863,6 +993,8 @@ const envSchema = rawSchema
checkRoleConsumesSomething(raw, ctx);
checkOtlpHasKey(raw, ctx);
checkKafkaHeartbeatFitsSession(raw, ctx);
+ checkKafkaSaslIsComplete(raw, ctx);
+ checkKafkaTlsOptionsHaveTls(raw, ctx);
checkReaperDeadmanOutlivesSession(raw, ctx);
checkVacuumOutlivesReaper(raw, ctx);
checkProfileBackfillProjectsHaveAFlag(raw, ctx);
diff --git a/docker-compose.yml b/docker-compose.yml
index 0dda0408b..e891fbbc0 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -64,6 +64,25 @@ services:
- "127.0.0.1:23092:19092" # Kafka API (external)
- "127.0.0.1:23644:9644" # Admin API
+ # Opt-in (profile "kafka-auth"): standalone broker that requires SASL, with a
+ # TLS listener, for verifying KAFKA_SSL/KAFKA_SASL_* support.
+ # See docker/redpanda/auth-entrypoint.sh. Not started by `bun run dock:up`.
+ op-rp-auth:
+ image: redpandadata/redpanda:v25.1.4
+ profiles: ["kafka-auth"]
+ entrypoint: ["/auth-entrypoint.sh"]
+ volumes:
+ - ./docker/data/op-rp-auth-certs:/certs
+ - ./docker/redpanda/auth-entrypoint.sh:/auth-entrypoint.sh:ro
+ ports:
+ - "127.0.0.1:23093:23093" # Kafka API, SASL required, plaintext
+ - "127.0.0.1:23094:23094" # Kafka API, SASL required, TLS
+ healthcheck:
+ test: ["CMD-SHELL", "rpk cluster health -X admin.hosts=localhost:9644 | grep -q 'Healthy:.*true'"]
+ interval: 5s
+ timeout: 3s
+ retries: 20
+
op-rp-console:
image: redpandadata/console:v3.7.2
restart: always
diff --git a/docker/redpanda/auth-entrypoint.sh b/docker/redpanda/auth-entrypoint.sh
new file mode 100755
index 000000000..5987bba6e
--- /dev/null
+++ b/docker/redpanda/auth-entrypoint.sh
@@ -0,0 +1,60 @@
+#!/usr/bin/env bash
+# Standalone Redpanda that REQUIRES SASL, for verifying the KAFKA_SSL /
+# KAFKA_SASL_* support in packages/core/src/modules/ingest against a real
+# broker. Not part of the default stack:
+#
+# bun run dock:kafka-auth # start (compose profile "kafka-auth")
+#
+# Listeners (both require SASL), bound to 127.0.0.1 by docker-compose.yml:
+# localhost:23093 plaintext
+# localhost:23094 TLS, self-signed CA written to docker/data/op-rp-auth-certs
+# Users: op512 (SCRAM-SHA-512), op256 (SCRAM-SHA-256); PLAIN works with both.
+set -euo pipefail
+
+CERT_DIR=/certs
+SASL_PORT=23093
+TLS_PORT=23094
+
+gen_certs() {
+ if [[ -f "$CERT_DIR/ca.crt" && -f "$CERT_DIR/broker.crt" && -f "$CERT_DIR/broker.key" ]]; then
+ return
+ fi
+ echo "generating self-signed TLS certs in $CERT_DIR"
+ openssl req -x509 -newkey rsa:2048 -nodes -keyout "$CERT_DIR/ca.key" -out "$CERT_DIR/ca.crt" \
+ -days 3650 -subj "/CN=openpanel-dev-ca" 2>/dev/null
+ openssl req -newkey rsa:2048 -nodes -keyout "$CERT_DIR/broker.key" -out "$CERT_DIR/broker.csr" \
+ -subj "/CN=localhost" 2>/dev/null
+ printf "subjectAltName=DNS:localhost,IP:127.0.0.1\n" > "$CERT_DIR/san.ext"
+ openssl x509 -req -in "$CERT_DIR/broker.csr" -CA "$CERT_DIR/ca.crt" -CAkey "$CERT_DIR/ca.key" \
+ -CAcreateserial -out "$CERT_DIR/broker.crt" -days 3650 -extfile "$CERT_DIR/san.ext" 2>/dev/null
+ chmod 644 "$CERT_DIR"/*.crt "$CERT_DIR"/*.key
+}
+
+configure_cluster() {
+ for _ in $(seq 1 60); do
+ if rpk cluster health -X admin.hosts=localhost:9644 >/dev/null 2>&1; then
+ break
+ fi
+ sleep 1
+ done
+ rpk cluster config set enable_sasl true -X admin.hosts=localhost:9644
+ rpk cluster config set sasl_mechanisms '["SCRAM","PLAIN"]' -X admin.hosts=localhost:9644
+ rpk cluster config set superusers '["op512","op256"]' -X admin.hosts=localhost:9644
+ # Redpanda stores one SCRAM mechanism per user; PLAIN works against either.
+ rpk security user create op512 -p op512-secret --mechanism SCRAM-SHA-512 -X admin.hosts=localhost:9644 || true
+ rpk security user create op256 -p op256-secret --mechanism SCRAM-SHA-256 -X admin.hosts=localhost:9644 || true
+ echo "kafka-auth broker ready"
+}
+
+gen_certs
+configure_cluster &
+
+exec rpk redpanda start \
+ --smp=1 \
+ --memory=512M \
+ --overprovisioned \
+ --mode=dev-container \
+ --default-log-level=warn \
+ --kafka-addr="sasl://0.0.0.0:$SASL_PORT,tls://0.0.0.0:$TLS_PORT" \
+ --advertise-kafka-addr="sasl://localhost:$SASL_PORT,tls://localhost:$TLS_PORT" \
+ --set "redpanda.kafka_api_tls=[{\"name\":\"tls\",\"enabled\":true,\"cert_file\":\"$CERT_DIR/broker.crt\",\"key_file\":\"$CERT_DIR/broker.key\",\"truststore_file\":\"$CERT_DIR/ca.crt\",\"require_client_auth\":false}]"
diff --git a/packages/core/src/config.ts b/packages/core/src/config.ts
index a61822b40..1ce57f3f3 100644
--- a/packages/core/src/config.ts
+++ b/packages/core/src/config.ts
@@ -85,6 +85,31 @@ export interface PolarConfig {
webhookSecret: string | undefined;
}
+export const KAFKA_SASL_MECHANISMS = [
+ 'plain',
+ 'scram-sha-256',
+ 'scram-sha-512',
+] as const;
+export type KafkaSaslMechanism = (typeof KAFKA_SASL_MECHANISMS)[number];
+
+/**
+ * TLS/SASL for an external broker. Unauthenticated plaintext when nothing is
+ * set. The loader has already checked the combination is coherent (a full
+ * credential pair, a known mechanism, TLS options only with TLS); the CA file
+ * itself is read where the client is built, so an unreadable path fails there
+ * with its own message.
+ */
+export interface KafkaSecurityConfig {
+ ssl: {
+ enabled: boolean;
+ caPath: string | undefined;
+ rejectUnauthorized: boolean | undefined;
+ };
+ sasl:
+ | { mechanism: KafkaSaslMechanism; username: string; password: string }
+ | undefined;
+}
+
/**
* The Kafka transport's knobs. Every one is resolved: kafkajs is handed these
* values verbatim, so a `??` at the call site would be a second default.
@@ -115,6 +140,7 @@ export interface KafkaConfig {
handlerMaxAttempts: number;
handlerRetryInitialMs: number;
handlerRetryMaxMs: number;
+ security: KafkaSecurityConfig;
}
/** Per-buffer sizing. `undefined` keeps the buffer's own documented default. */
diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts
index 459cc168e..fccddad7f 100644
--- a/packages/core/src/index.ts
+++ b/packages/core/src/index.ts
@@ -9,7 +9,14 @@
// packages/db reads ClickHouse row JSON through this (P11: packages/json's
// only definition already lived here; the barrel is how a package outside
// core reaches it, since the exports map has no ./shared/* entry).
-export { generateId, generateSecureId, getChartPrevStartEndDate, getSafeJson, resolveDateRange, shortId } from '@openpanel/shared';
+export {
+ generateId,
+ generateSecureId,
+ getChartPrevStartEndDate,
+ getSafeJson,
+ resolveDateRange,
+ shortId,
+} from '@openpanel/shared';
export type {
SafeFetchOptions,
SafeFetchResult,
@@ -23,21 +30,25 @@ export type {
// relative import now that it lives inside core (M5-007).
export {
assertPublicHostname,
- assertPublicUrl,assertSafeUrl,
+ assertPublicUrl,
+ assertSafeUrl,
BlockedUrlError,
createHash,
- createPinnedAgent,createPinnedLookup,
+ createPinnedAgent,
+ createPinnedLookup,
decrypt,
decryptCredential,
encrypt,
encryptCredential,
- generateSalt,getDevice,
+ generateSalt,
+ getDevice,
hashPassword,
isBlockedIp,
- isEncrypted,parseUserAgent,
+ isEncrypted,
+ parseUserAgent,
safeFetch,
- safeFetchStream,
- verifyPassword
+ safeFetchStream,
+ verifyPassword,
} from '@openpanel/shared/server';
// The seven buffers (moved from packages/db/src/buffers, M8-001). Only the
// FACTORY is on the barrel: they are boot singletons on `AppDeps`, built once
@@ -77,6 +88,10 @@ export {
ProviderError,
type ProviderErrorOptions,
} from './clients/provider-error';
+// The one config VALUE the loader needs from core: the accepted SASL
+// mechanisms, so `KAFKA_SASL_MECHANISM` is validated against the list the
+// client actually supports rather than a second copy of it.
+export { KAFKA_SASL_MECHANISMS, type KafkaSaslMechanism } from './config';
export type {
AppDeps,
CoreConfig,
diff --git a/packages/core/src/modules/ingest/src/kafka-security.test.ts b/packages/core/src/modules/ingest/src/kafka-security.test.ts
new file mode 100644
index 000000000..0bcebf734
--- /dev/null
+++ b/packages/core/src/modules/ingest/src/kafka-security.test.ts
@@ -0,0 +1,117 @@
+import { describe, expect, it } from 'bun:test';
+import type { KafkaSecurityConfig } from '../../../config';
+import { describeKafkaSecurity, resolveKafkaSecurity } from './kafka-security';
+
+const FAKE_PEM = '-----BEGIN CERTIFICATE-----\nfake\n-----END CERTIFICATE-----';
+
+const readFile = (path: string): string => {
+ if (path === '/certs/ca.pem') {
+ return FAKE_PEM;
+ }
+ throw new Error(`ENOENT: no such file or directory, open '${path}'`);
+};
+
+const plaintext: KafkaSecurityConfig = {
+ ssl: { enabled: false, caPath: undefined, rejectUnauthorized: undefined },
+ sasl: undefined,
+};
+
+const withSasl = (
+ mechanism: 'plain' | 'scram-sha-256' | 'scram-sha-512',
+ ssl: KafkaSecurityConfig['ssl'] = {
+ enabled: true,
+ caPath: undefined,
+ rejectUnauthorized: undefined,
+ }
+): KafkaSecurityConfig => ({
+ ssl,
+ sasl: { mechanism, username: 'op', password: 'secret' },
+});
+
+describe('resolveKafkaSecurity', () => {
+ it('is plaintext without auth when nothing is configured', () => {
+ expect(resolveKafkaSecurity(plaintext, readFile)).toEqual({
+ ssl: false,
+ sasl: undefined,
+ });
+ });
+
+ it('is a bare `true` for TLS with the system trust store', () => {
+ expect(
+ resolveKafkaSecurity(
+ { ...plaintext, ssl: { ...plaintext.ssl, enabled: true } },
+ readFile
+ )
+ ).toEqual({ ssl: true, sasl: undefined });
+ });
+
+ it('passes a custom CA and rejectUnauthorized through to tls options', () => {
+ expect(
+ resolveKafkaSecurity(
+ {
+ ...plaintext,
+ ssl: {
+ enabled: true,
+ caPath: '/certs/ca.pem',
+ rejectUnauthorized: false,
+ },
+ },
+ readFile
+ ).ssl
+ ).toEqual({ ca: [FAKE_PEM], rejectUnauthorized: false });
+ });
+
+ it('fails with the path when the CA file cannot be read', () => {
+ expect(() =>
+ resolveKafkaSecurity(
+ {
+ ...plaintext,
+ ssl: {
+ enabled: true,
+ caPath: '/missing.pem',
+ rejectUnauthorized: undefined,
+ },
+ },
+ readFile
+ )
+ ).toThrow(/Failed to read KAFKA_SSL_CA_PATH "\/missing.pem"/);
+ });
+
+ it('hands SASL credentials to kafkajs unchanged, per mechanism', () => {
+ for (const mechanism of [
+ 'plain',
+ 'scram-sha-256',
+ 'scram-sha-512',
+ ] as const) {
+ expect(resolveKafkaSecurity(withSasl(mechanism), readFile)).toEqual({
+ ssl: true,
+ sasl: { mechanism, username: 'op', password: 'secret' },
+ });
+ }
+ });
+
+ it('allows SASL over plaintext when the loader resolved TLS off', () => {
+ expect(
+ resolveKafkaSecurity(withSasl('plain', plaintext.ssl), readFile)
+ ).toEqual({
+ ssl: false,
+ sasl: { mechanism: 'plain', username: 'op', password: 'secret' },
+ });
+ });
+});
+
+describe('describeKafkaSecurity', () => {
+ it('summarises without credential values', () => {
+ const resolved = resolveKafkaSecurity(withSasl('scram-sha-512'), readFile);
+ expect(describeKafkaSecurity(resolved)).toEqual({
+ ssl: true,
+ sasl: 'scram-sha-512',
+ });
+ expect(JSON.stringify(describeKafkaSecurity(resolved))).not.toContain(
+ 'secret'
+ );
+ expect(
+ describeKafkaSecurity(resolveKafkaSecurity(plaintext, readFile))
+ ).toEqual({ ssl: false, sasl: null });
+ });
+});
diff --git a/packages/core/src/modules/ingest/src/kafka-security.ts b/packages/core/src/modules/ingest/src/kafka-security.ts
new file mode 100644
index 000000000..d3ddab5be
--- /dev/null
+++ b/packages/core/src/modules/ingest/src/kafka-security.ts
@@ -0,0 +1,62 @@
+// The kafkajs `ssl` / `sasl` options for an external broker, built from the
+// already-validated `KafkaSecurityConfig`. The only thing that can still fail
+// here is reading the CA file, and it fails with the path in the message.
+
+import { readFileSync } from 'node:fs';
+import type { ConnectionOptions } from 'node:tls';
+import type { SASLOptions } from 'kafkajs';
+import type { KafkaSaslMechanism, KafkaSecurityConfig } from '../../../config';
+
+export interface ResolvedKafkaSecurity {
+ ssl: ConnectionOptions | boolean;
+ sasl: SASLOptions | undefined;
+}
+
+export type ReadPemFile = (path: string) => string;
+
+const readPemFile: ReadPemFile = (path) => readFileSync(path, 'utf8');
+
+const resolveSsl = (
+ ssl: KafkaSecurityConfig['ssl'],
+ readFile: ReadPemFile
+): ConnectionOptions | boolean => {
+ if (!ssl.enabled) {
+ return false;
+ }
+ const options: ConnectionOptions = {};
+ if (ssl.caPath) {
+ try {
+ options.ca = [readFile(ssl.caPath)];
+ } catch (err) {
+ throw new Error(
+ `Failed to read KAFKA_SSL_CA_PATH "${ssl.caPath}": ${err instanceof Error ? err.message : String(err)}`
+ );
+ }
+ }
+ if (ssl.rejectUnauthorized !== undefined) {
+ options.rejectUnauthorized = ssl.rejectUnauthorized;
+ }
+ return Object.keys(options).length > 0 ? options : true;
+};
+
+export const resolveKafkaSecurity = (
+ security: KafkaSecurityConfig,
+ readFile: ReadPemFile = readPemFile
+): ResolvedKafkaSecurity => ({
+ ssl: resolveSsl(security.ssl, readFile),
+ sasl: security.sasl
+ ? {
+ mechanism: security.sasl.mechanism,
+ username: security.sasl.username,
+ password: security.sasl.password,
+ }
+ : undefined,
+});
+
+/** Log-safe summary: never includes credential values. */
+export const describeKafkaSecurity = (
+ security: ResolvedKafkaSecurity
+): { ssl: boolean; sasl: KafkaSaslMechanism | null } => ({
+ ssl: security.ssl !== false,
+ sasl: security.sasl ? (security.sasl.mechanism as KafkaSaslMechanism) : null,
+});
diff --git a/packages/core/src/modules/ingest/src/kafka.ts b/packages/core/src/modules/ingest/src/kafka.ts
index 02c75f6f1..437224ce0 100644
--- a/packages/core/src/modules/ingest/src/kafka.ts
+++ b/packages/core/src/modules/ingest/src/kafka.ts
@@ -25,6 +25,11 @@ import { createLogger, type ILogger } from '../../../pino-logger';
// message shape it hands to this producer, and both files are now siblings.
import type { DeadLetterMessage } from './consumer';
import type { IncomingEventPayload } from './incoming-event';
+import {
+ describeKafkaSecurity,
+ type ResolvedKafkaSecurity,
+ resolveKafkaSecurity,
+} from './kafka-security';
import {
createProducerBatcher,
type ProducerBatcher,
@@ -59,18 +64,31 @@ export const assertKafkaConfigured = (config: KafkaConfig): void => {
'KAFKA_BROKERS is not set. Kafka/Redpanda is the only events transport — set KAFKA_BROKERS to a comma-separated broker list.'
);
}
+ // Reads the CA file, so an unreadable KAFKA_SSL_CA_PATH fails here at boot
+ // rather than on the first produce/consume.
+ getKafkaSecurity(config);
+};
+
+// Memoised like the client below: the CA file is read once per process.
+let kafkaSecurity: ResolvedKafkaSecurity | null = null;
+const getKafkaSecurity = (config: KafkaConfig): ResolvedKafkaSecurity => {
+ kafkaSecurity ??= resolveKafkaSecurity(config.security);
+ return kafkaSecurity;
};
let kafka: Kafka | null = null;
const getKafka = (config: CoreConfig): Kafka => {
assertKafkaConfigured(config.kafka);
if (!kafka) {
+ const security = getKafkaSecurity(config.kafka);
kafka = new Kafka({
clientId: config.kafka.clientId,
brokers: config.kafka.brokers,
logLevel: logLevel.WARN,
requestTimeout: config.kafka.requestTimeoutMs,
connectionTimeout: config.kafka.connectionTimeoutMs,
+ ssl: security.ssl,
+ sasl: security.sasl,
});
}
return kafka;
@@ -112,6 +130,8 @@ const getProducer = async (config: CoreConfig): Promise<Producer> => {
{
brokers: config.kafka.brokers,
topic: config.kafka.eventsTopic,
+ // `ssl: bool` and the mechanism only — never a credential.
+ ...describeKafkaSecurity(getKafkaSecurity(config.kafka)),
// The throughput knobs, logged once, so a measurement run can be
// tied to the configuration that produced it.
maxInFlight: tuning.maxInFlight,
diff --git a/packages/core/src/modules/ingest/src/producer-tuning.test.ts b/packages/core/src/modules/ingest/src/producer-tuning.test.ts
index c1f305a0b..f60d70bfb 100644
--- a/packages/core/src/modules/ingest/src/producer-tuning.test.ts
+++ b/packages/core/src/modules/ingest/src/producer-tuning.test.ts
@@ -38,6 +38,10 @@ const kafkaConfig = (tuning: Partial<KafkaConfig> = {}): KafkaConfig =>
handlerMaxAttempts: 3,
handlerRetryInitialMs: 100,
handlerRetryMaxMs: 1000,
+ security: {
+ ssl: { enabled: false, caPath: undefined, rejectUnauthorized: undefined },
+ sasl: undefined,
+ },
...tuning,
}) satisfies KafkaConfig;
diff --git a/packages/core/test/config-fixture.ts b/packages/core/test/config-fixture.ts
index 5a7fa0307..8f30614a6 100644
--- a/packages/core/test/config-fixture.ts
+++ b/packages/core/test/config-fixture.ts
@@ -81,6 +81,14 @@ export function testCoreConfig(
handlerMaxAttempts: 3,
handlerRetryInitialMs: 100,
handlerRetryMaxMs: 1000,
+ security: {
+ ssl: {
+ enabled: false,
+ caPath: undefined,
+ rejectUnauthorized: undefined,
+ },
+ sasl: undefined,
+ },
},
buffers: {
asyncInserts: false,