Skip to content

OpenPanel is an open-source web and product analytics platform, an open-source alternative to Mixpanel with optional self-hosting.

feat(kafka): support TLS and SASL auth for external brokers

Carl-Gerhard Lindesvärdf80b12411269
diff --git a/apps/public/content/docs/self-hosting/environment-variables.mdx b/apps/public/content/docs/self-hosting/environment-variables.mdx
index bb4442ebf..9b1d3fc5e 100644
--- a/apps/public/content/docs/self-hosting/environment-variables.mdx
+++ b/apps/public/content/docs/self-hosting/environment-variables.mdx
@@ -742,6 +742,8 @@ OpenPanel can optionally ingest events through a Kafka-compatible broker (Kafka
Routing is all-or-nothing: if `KAFKA_BROKERS` is set, every event is produced to the Kafka topic; otherwise all events stay on the default Redis-backed `events` queue. Set `ENABLED_QUEUES=events_kafka` on the worker(s) that should consume from Kafka.
+Connections are unauthenticated plaintext by default. For managed or production clusters, enable TLS with `KAFKA_SSL` and SASL authentication (`plain`, `scram-sha-256` or `scram-sha-512`) with the `KAFKA_SASL_*` variables below.
+
<Callout>
The client speaks the Kafka wire protocol via [kafkajs](https://kafka.js.org/), so any Kafka-compatible broker works. The bundled `docker-compose.yml` uses [Redpanda](https://redpanda.com/) as a lightweight, single-binary option, but you can point `KAFKA_BROKERS` at any Kafka cluster.
</Callout>
@@ -761,6 +763,98 @@ KAFKA_BROKERS=redpanda:9092
KAFKA_BROKERS=kafka-1:9092,kafka-2:9092,kafka-3:9092
```
+### KAFKA_SSL
+
+**Type**: `boolean`
+**Required**: No
+**Default**: `false` (automatically `true` when SASL credentials are set)
+
+Connect to the brokers over TLS. Set this to `true` for a TLS-only cluster without authentication. When `KAFKA_SASL_USERNAME`/`KAFKA_SASL_PASSWORD` are set, TLS is enabled automatically; set `KAFKA_SSL=false` explicitly to send SASL over plaintext (only sensible on a trusted internal network).
+
+**Example**:
+```bash
+KAFKA_SSL=true
+```
+
+### KAFKA_SSL_CA_PATH
+
+**Type**: `string`
+**Required**: No
+**Default**: None (system trust store)
+
+Path to a PEM-encoded CA certificate (or bundle) to trust in addition to the system trust store. Use this when your brokers present certificates signed by a private or self-signed CA. Requires TLS to be enabled.
+
+**Example**:
+```bash
+KAFKA_SSL_CA_PATH=/etc/openpanel/kafka-ca.pem
+```
+
+### KAFKA_SSL_REJECT_UNAUTHORIZED
+
+**Type**: `boolean`
+**Required**: No
+**Default**: `true`
+
+Whether to reject broker certificates that cannot be verified. Only set to `false` for local development against a self-signed broker; prefer `KAFKA_SSL_CA_PATH` in production. Requires TLS to be enabled.
+
+**Example**:
+```bash
+KAFKA_SSL_REJECT_UNAUTHORIZED=false
+```
+
+### KAFKA_SASL_USERNAME
+
+**Type**: `string`
+**Required**: No (required together with `KAFKA_SASL_PASSWORD` to enable SASL)
+**Default**: None
+
+SASL username. Setting a username without a password (or vice versa) is a configuration error and the process refuses to start. Credential values are never logged.
+
+**Example**:
+```bash
+KAFKA_SASL_USERNAME=openpanel
+```
+
+### KAFKA_SASL_PASSWORD
+
+**Type**: `string`
+**Required**: No (required together with `KAFKA_SASL_USERNAME` to enable SASL)
+**Default**: None
+
+SASL password.
+
+**Example**:
+```bash
+KAFKA_SASL_PASSWORD=your-secret
+```
+
+### KAFKA_SASL_MECHANISM
+
+**Type**: `string`
+**Required**: No
+**Default**: `scram-sha-512`
+
+SASL mechanism to authenticate with. One of `plain`, `scram-sha-256` or `scram-sha-512` (case-insensitive). Any other value is rejected at startup, before a Kafka client is created.
+
+**Example**:
+```bash
+# TLS + SASL/SCRAM-SHA-512 (the default when credentials are set)
+KAFKA_BROKERS=kafka-1.example.com:9093,kafka-2.example.com:9093
+KAFKA_SASL_USERNAME=openpanel
+KAFKA_SASL_PASSWORD=your-secret
+
+# TLS + SASL/PLAIN with a private CA
+KAFKA_BROKERS=kafka.internal:9093
+KAFKA_SASL_MECHANISM=plain
+KAFKA_SASL_USERNAME=openpanel
+KAFKA_SASL_PASSWORD=your-secret
+KAFKA_SSL_CA_PATH=/etc/openpanel/kafka-ca.pem
+```
+
+<Callout>
+The producer runs in idempotent mode, so the SASL principal needs the `IdempotentWrite` cluster permission in addition to `Write`/`Describe` on the topic and `Read` on the consumer group. Managed Kafka providers usually grant this by default; on a locked-down cluster you may need to add it explicitly.
+</Callout>
+
### KAFKA_EVENTS_TOPIC
**Type**: `string`
diff --git a/docker-compose.yml b/docker-compose.yml
index 189574789..782709a01 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -62,6 +62,25 @@ services:
- "19092:19092" # Kafka API (external)
- "9644:9644" # Admin API
+ # Opt-in (profile "kafka-auth"): standalone broker that requires SASL, with a
+ # TLS listener, for verifying KAFKA_SSL/KAFKA_SASL_* support.
+ # See docker/redpanda/auth-entrypoint.sh. Not started by `pnpm dock:up`.
+ op-rp-auth:
+ image: redpandadata/redpanda:v25.1.4
+ profiles: ["kafka-auth"]
+ entrypoint: ["/auth-entrypoint.sh"]
+ volumes:
+ - ./docker/data/op-rp-auth-certs:/certs
+ - ./docker/redpanda/auth-entrypoint.sh:/auth-entrypoint.sh:ro
+ ports:
+ - "19093:19093" # Kafka API, SASL required, plaintext
+ - "19094:19094" # Kafka API, SASL required, TLS
+ healthcheck:
+ test: ["CMD-SHELL", "rpk cluster health -X admin.hosts=localhost:9644 | grep -q 'Healthy:.*true'"]
+ interval: 5s
+ timeout: 3s
+ retries: 20
+
op-rp-console:
image: redpandadata/console:v3.7.2
restart: always
diff --git a/docker/redpanda/auth-entrypoint.sh b/docker/redpanda/auth-entrypoint.sh
new file mode 100755
index 000000000..f9aa090b3
--- /dev/null
+++ b/docker/redpanda/auth-entrypoint.sh
@@ -0,0 +1,60 @@
+#!/usr/bin/env bash
+# Standalone Redpanda that REQUIRES SASL, for verifying the TLS/SASL support in
+# packages/queue against a real broker. Not part of the default stack:
+#
+# pnpm dock:kafka-auth # start (compose profile "kafka-auth")
+# pnpm vitest run packages/queue/src/kafka.integration.test.ts
+#
+# Listeners (both require SASL):
+# localhost:19093 plaintext
+# localhost:19094 TLS, self-signed CA written to docker/data/op-rp-auth-certs
+# Users: op512 (SCRAM-SHA-512), op256 (SCRAM-SHA-256); PLAIN works with both.
+set -euo pipefail
+
+CERT_DIR=/certs
+SASL_PORT=19093
+TLS_PORT=19094
+
+gen_certs() {
+ if [[ -f "$CERT_DIR/ca.crt" && -f "$CERT_DIR/broker.crt" && -f "$CERT_DIR/broker.key" ]]; then
+ return
+ fi
+ echo "generating self-signed TLS certs in $CERT_DIR"
+ openssl req -x509 -newkey rsa:2048 -nodes -keyout "$CERT_DIR/ca.key" -out "$CERT_DIR/ca.crt" \
+ -days 3650 -subj "/CN=openpanel-dev-ca" 2>/dev/null
+ openssl req -newkey rsa:2048 -nodes -keyout "$CERT_DIR/broker.key" -out "$CERT_DIR/broker.csr" \
+ -subj "/CN=localhost" 2>/dev/null
+ printf "subjectAltName=DNS:localhost,IP:127.0.0.1\n" > "$CERT_DIR/san.ext"
+ openssl x509 -req -in "$CERT_DIR/broker.csr" -CA "$CERT_DIR/ca.crt" -CAkey "$CERT_DIR/ca.key" \
+ -CAcreateserial -out "$CERT_DIR/broker.crt" -days 3650 -extfile "$CERT_DIR/san.ext" 2>/dev/null
+ chmod 644 "$CERT_DIR"/*.crt "$CERT_DIR"/*.key
+}
+
+configure_cluster() {
+ for _ in $(seq 1 60); do
+ if rpk cluster health -X admin.hosts=localhost:9644 >/dev/null 2>&1; then
+ break
+ fi
+ sleep 1
+ done
+ rpk cluster config set enable_sasl true -X admin.hosts=localhost:9644
+ rpk cluster config set sasl_mechanisms '["SCRAM","PLAIN"]' -X admin.hosts=localhost:9644
+ rpk cluster config set superusers '["op512","op256"]' -X admin.hosts=localhost:9644
+ # Redpanda stores one SCRAM mechanism per user; PLAIN works against either.
+ rpk security user create op512 -p op512-secret --mechanism SCRAM-SHA-512 -X admin.hosts=localhost:9644 || true
+ rpk security user create op256 -p op256-secret --mechanism SCRAM-SHA-256 -X admin.hosts=localhost:9644 || true
+ echo "kafka-auth broker ready"
+}
+
+gen_certs
+configure_cluster &
+
+exec rpk redpanda start \
+ --smp=1 \
+ --memory=512M \
+ --overprovisioned \
+ --mode=dev-container \
+ --default-log-level=warn \
+ --kafka-addr="sasl://0.0.0.0:$SASL_PORT,tls://0.0.0.0:$TLS_PORT" \
+ --advertise-kafka-addr="sasl://localhost:$SASL_PORT,tls://localhost:$TLS_PORT" \
+ --set "redpanda.kafka_api_tls=[{\"name\":\"tls\",\"enabled\":true,\"cert_file\":\"$CERT_DIR/broker.crt\",\"key_file\":\"$CERT_DIR/broker.key\",\"truststore_file\":\"$CERT_DIR/ca.crt\",\"require_client_auth\":false}]"
diff --git a/package.json b/package.json
index 8d3681acf..5978fe7dc 100644
--- a/package.json
+++ b/package.json
@@ -12,6 +12,7 @@
"gen:referrers": "pnpm -r --filter common gen:referrers",
"dock:up": "docker compose up -d",
"dock:down": "docker compose down",
+ "dock:kafka-auth": "docker compose --profile kafka-auth up -d --wait op-rp-auth",
"dock:ch": "docker compose exec -it op-ch clickhouse-client -d openpanel",
"dock:redis": "docker compose exec -it op-kv redis-cli",
"db:codegen": "pnpm -r --filter db run codegen",
diff --git a/packages/queue/package.json b/packages/queue/package.json
index 989f30f48..33df9e692 100644
--- a/packages/queue/package.json
+++ b/packages/queue/package.json
@@ -4,7 +4,8 @@
"type": "module",
"main": "index.ts",
"scripts": {
- "typecheck": "tsc --noEmit"
+ "typecheck": "tsc --noEmit",
+ "test": "vitest run"
},
"dependencies": {
"@openpanel/db": "workspace:*",
diff --git a/packages/queue/src/kafka-security.test.ts b/packages/queue/src/kafka-security.test.ts
new file mode 100644
index 000000000..d6ef92592
--- /dev/null
+++ b/packages/queue/src/kafka-security.test.ts
@@ -0,0 +1,180 @@
+import { describe, expect, it } from 'vitest';
+import { describeKafkaSecurity, resolveKafkaSecurity } from './kafka-security';
+
+const readFile = (path: string): string => {
+ if (path === '/certs/ca.pem') {
+ return '-----BEGIN CERTIFICATE-----\nfake\n-----END CERTIFICATE-----';
+ }
+ throw new Error(`ENOENT: no such file or directory, open '${path}'`);
+};
+
+describe('resolveKafkaSecurity', () => {
+ it('defaults to plaintext without auth when nothing is configured', () => {
+ expect(resolveKafkaSecurity({}, readFile)).toEqual({
+ ssl: false,
+ sasl: undefined,
+ });
+ });
+
+ it('enables TLS only with KAFKA_SSL=true', () => {
+ expect(resolveKafkaSecurity({ KAFKA_SSL: 'true' }, readFile)).toEqual({
+ ssl: true,
+ sasl: undefined,
+ });
+ expect(resolveKafkaSecurity({ KAFKA_SSL: '1' }, readFile).ssl).toBe(true);
+ });
+
+ it('passes a custom CA and rejectUnauthorized through to tls options', () => {
+ expect(
+ resolveKafkaSecurity(
+ {
+ KAFKA_SSL: 'true',
+ KAFKA_SSL_CA_PATH: '/certs/ca.pem',
+ KAFKA_SSL_REJECT_UNAUTHORIZED: 'false',
+ },
+ readFile
+ ).ssl
+ ).toEqual({
+ ca: ['-----BEGIN CERTIFICATE-----\nfake\n-----END CERTIFICATE-----'],
+ rejectUnauthorized: false,
+ });
+ });
+
+ it('fails with a clear error when the CA file cannot be read', () => {
+ expect(() =>
+ resolveKafkaSecurity(
+ { KAFKA_SSL: 'true', KAFKA_SSL_CA_PATH: '/missing.pem' },
+ readFile
+ )
+ ).toThrow(/Failed to read KAFKA_SSL_CA_PATH "\/missing.pem"/);
+ });
+
+ it('rejects TLS-only options when TLS is disabled', () => {
+ expect(() =>
+ resolveKafkaSecurity({ KAFKA_SSL_CA_PATH: '/certs/ca.pem' }, readFile)
+ ).toThrow(/require TLS/);
+ expect(() =>
+ resolveKafkaSecurity(
+ { KAFKA_SSL: 'false', KAFKA_SSL_REJECT_UNAUTHORIZED: 'false' },
+ readFile
+ )
+ ).toThrow(/require TLS/);
+ });
+
+ it('rejects non-boolean KAFKA_SSL values', () => {
+ expect(() => resolveKafkaSecurity({ KAFKA_SSL: 'yes' }, readFile)).toThrow(
+ /KAFKA_SSL must be "true" or "false"/
+ );
+ });
+
+ it('defaults SASL to scram-sha-512 over TLS when credentials are present', () => {
+ expect(
+ resolveKafkaSecurity(
+ { KAFKA_SASL_USERNAME: 'op', KAFKA_SASL_PASSWORD: 'secret' },
+ readFile
+ )
+ ).toEqual({
+ ssl: true,
+ sasl: { mechanism: 'scram-sha-512', username: 'op', password: 'secret' },
+ });
+ });
+
+ it.each([
+ 'plain',
+ 'scram-sha-256',
+ 'scram-sha-512',
+ ] as const)('supports mechanism %s (case-insensitive)', (mechanism) => {
+ const config = resolveKafkaSecurity(
+ {
+ KAFKA_SASL_USERNAME: 'op',
+ KAFKA_SASL_PASSWORD: 'secret',
+ KAFKA_SASL_MECHANISM: mechanism.toUpperCase(),
+ },
+ readFile
+ );
+ expect(config.sasl?.mechanism).toBe(mechanism);
+ expect(config.ssl).toBe(true);
+ });
+
+ it('allows SASL over plaintext when KAFKA_SSL=false is explicit', () => {
+ expect(
+ resolveKafkaSecurity(
+ {
+ KAFKA_SSL: 'false',
+ KAFKA_SASL_USERNAME: 'op',
+ KAFKA_SASL_PASSWORD: 'secret',
+ KAFKA_SASL_MECHANISM: 'plain',
+ },
+ readFile
+ )
+ ).toEqual({
+ ssl: false,
+ sasl: { mechanism: 'plain', username: 'op', password: 'secret' },
+ });
+ });
+
+ it('rejects unsupported mechanisms before creating a client', () => {
+ expect(() =>
+ resolveKafkaSecurity(
+ {
+ KAFKA_SASL_USERNAME: 'op',
+ KAFKA_SASL_PASSWORD: 'secret',
+ KAFKA_SASL_MECHANISM: 'oauthbearer',
+ },
+ readFile
+ )
+ ).toThrow(
+ 'Unsupported KAFKA_SASL_MECHANISM "oauthbearer". Supported: plain, scram-sha-256, scram-sha-512'
+ );
+ });
+
+ it('fails fast when only one of username/password is set', () => {
+ expect(() =>
+ resolveKafkaSecurity({ KAFKA_SASL_USERNAME: 'op' }, readFile)
+ ).toThrow(/KAFKA_SASL_PASSWORD is missing/);
+ expect(() =>
+ resolveKafkaSecurity({ KAFKA_SASL_PASSWORD: 'secret' }, readFile)
+ ).toThrow(/KAFKA_SASL_USERNAME is missing/);
+ });
+
+ it('fails fast when a mechanism is set without credentials', () => {
+ expect(() =>
+ resolveKafkaSecurity({ KAFKA_SASL_MECHANISM: 'plain' }, readFile)
+ ).toThrow(/KAFKA_SASL_MECHANISM is set but/);
+ });
+
+ it('never leaks credentials in error messages', () => {
+ const env = {
+ KAFKA_SASL_USERNAME: 'op',
+ KAFKA_SASL_PASSWORD: 'hunter2-secret',
+ KAFKA_SASL_MECHANISM: 'nope',
+ };
+ let message = '';
+ try {
+ resolveKafkaSecurity(env, readFile);
+ } catch (err) {
+ message = err instanceof Error ? err.message : String(err);
+ }
+ expect(message).not.toContain('hunter2-secret');
+ });
+});
+
+describe('describeKafkaSecurity', () => {
+ it('summarises without credential values', () => {
+ const config = resolveKafkaSecurity(
+ { KAFKA_SASL_USERNAME: 'op', KAFKA_SASL_PASSWORD: 'secret' },
+ readFile
+ );
+ expect(describeKafkaSecurity(config)).toEqual({
+ ssl: true,
+ sasl: 'scram-sha-512',
+ });
+ expect(JSON.stringify(describeKafkaSecurity(config))).not.toContain(
+ 'secret'
+ );
+ expect(describeKafkaSecurity(resolveKafkaSecurity({}, readFile))).toEqual({
+ ssl: false,
+ sasl: null,
+ });
+ });
+});
diff --git a/packages/queue/src/kafka-security.ts b/packages/queue/src/kafka-security.ts
new file mode 100644
index 000000000..a83cd5ce6
--- /dev/null
+++ b/packages/queue/src/kafka-security.ts
@@ -0,0 +1,150 @@
+import { readFileSync } from 'node:fs';
+import type { ConnectionOptions } from 'node:tls';
+import type { SASLOptions } from 'kafkajs';
+
+// Env-driven TLS/SASL config for external Kafka brokers. Kept free of any
+// project imports so it can be unit-tested in isolation.
+//
+// Supported modes (all opt-in via env, unauthenticated plaintext by default):
+// - TLS only KAFKA_SSL=true
+// - SASL/PLAIN over TLS KAFKA_SASL_MECHANISM=plain + credentials
+// - SASL/SCRAM-SHA-256 over TLS KAFKA_SASL_MECHANISM=scram-sha-256 + credentials
+// - SASL/SCRAM-SHA-512 over TLS KAFKA_SASL_MECHANISM=scram-sha-512 + credentials (default)
+// TLS is implied when SASL credentials are present; set KAFKA_SSL=false to
+// send SASL over plaintext (dev/internal networks only).
+
+export const KAFKA_SASL_MECHANISMS = [
+ 'plain',
+ 'scram-sha-256',
+ 'scram-sha-512',
+] as const;
+export type KafkaSaslMechanism = (typeof KAFKA_SASL_MECHANISMS)[number];
+const DEFAULT_SASL_MECHANISM: KafkaSaslMechanism = 'scram-sha-512';
+
+// Shape-compatible with process.env. Keys read:
+// KAFKA_SSL, KAFKA_SSL_CA_PATH, KAFKA_SSL_REJECT_UNAUTHORIZED,
+// KAFKA_SASL_USERNAME, KAFKA_SASL_PASSWORD, KAFKA_SASL_MECHANISM
+export type KafkaSecurityEnv = Readonly<Record<string, string | undefined>>;
+
+export type KafkaSecurityConfig = {
+ ssl: ConnectionOptions | boolean;
+ sasl: SASLOptions | undefined;
+};
+
+const isSaslMechanism = (value: string): value is KafkaSaslMechanism =>
+ (KAFKA_SASL_MECHANISMS as readonly string[]).includes(value);
+
+const parseBoolean = (
+ name: string,
+ raw: string | undefined
+): boolean | undefined => {
+ if (raw === undefined || raw.trim() === '') {
+ return undefined;
+ }
+ const value = raw.trim().toLowerCase();
+ if (value === 'true' || value === '1') {
+ return true;
+ }
+ if (value === 'false' || value === '0') {
+ return false;
+ }
+ throw new Error(`${name} must be "true" or "false" (got "${raw}")`);
+};
+
+const resolveSasl = (env: KafkaSecurityEnv): SASLOptions | undefined => {
+ const username = env.KAFKA_SASL_USERNAME;
+ const password = env.KAFKA_SASL_PASSWORD;
+ const hasUsername = Boolean(username);
+ const hasPassword = Boolean(password);
+
+ if (!(hasUsername || hasPassword)) {
+ if (env.KAFKA_SASL_MECHANISM) {
+ throw new Error(
+ 'KAFKA_SASL_MECHANISM is set but KAFKA_SASL_USERNAME and KAFKA_SASL_PASSWORD are missing'
+ );
+ }
+ return undefined;
+ }
+ if (!(hasUsername && hasPassword)) {
+ const missing = hasUsername ? 'KAFKA_SASL_PASSWORD' : 'KAFKA_SASL_USERNAME';
+ throw new Error(
+ `Kafka SASL is partially configured: ${missing} is missing (both KAFKA_SASL_USERNAME and KAFKA_SASL_PASSWORD are required)`
+ );
+ }
+
+ const rawMechanism = env.KAFKA_SASL_MECHANISM?.trim().toLowerCase();
+ const mechanism = rawMechanism || DEFAULT_SASL_MECHANISM;
+ if (!isSaslMechanism(mechanism)) {
+ throw new Error(
+ `Unsupported KAFKA_SASL_MECHANISM "${env.KAFKA_SASL_MECHANISM}". Supported: ${KAFKA_SASL_MECHANISMS.join(', ')}`
+ );
+ }
+
+ return {
+ mechanism,
+ username: username as string,
+ password: password as string,
+ };
+};
+
+const resolveSsl = (
+ env: KafkaSecurityEnv,
+ saslEnabled: boolean,
+ readFile: (path: string) => string
+): ConnectionOptions | boolean => {
+ // SASL credentials go over TLS unless explicitly opted out.
+ const enabled = parseBoolean('KAFKA_SSL', env.KAFKA_SSL) ?? saslEnabled;
+ const caPath = env.KAFKA_SSL_CA_PATH?.trim();
+ const rejectUnauthorized = parseBoolean(
+ 'KAFKA_SSL_REJECT_UNAUTHORIZED',
+ env.KAFKA_SSL_REJECT_UNAUTHORIZED
+ );
+
+ if (!enabled) {
+ if (caPath || rejectUnauthorized !== undefined) {
+ throw new Error(
+ 'KAFKA_SSL_CA_PATH / KAFKA_SSL_REJECT_UNAUTHORIZED require TLS; set KAFKA_SSL=true'
+ );
+ }
+ return false;
+ }
+
+ const options: ConnectionOptions = {};
+ if (caPath) {
+ try {
+ options.ca = [readFile(caPath)];
+ } catch (err) {
+ throw new Error(
+ `Failed to read KAFKA_SSL_CA_PATH "${caPath}": ${err instanceof Error ? err.message : String(err)}`
+ );
+ }
+ }
+ if (rejectUnauthorized !== undefined) {
+ options.rejectUnauthorized = rejectUnauthorized;
+ }
+ return Object.keys(options).length > 0 ? options : true;
+};
+
+const readPemFile = (path: string): string => readFileSync(path, 'utf8');
+
+/**
+ * Build the kafkajs `ssl` / `sasl` options from env. Throws a descriptive
+ * Error on inconsistent configuration so misconfiguration surfaces at
+ * startup instead of as an opaque broker handshake failure.
+ */
+export const resolveKafkaSecurity = (
+ env: KafkaSecurityEnv,
+ readFile: (path: string) => string = readPemFile
+): KafkaSecurityConfig => {
+ const sasl = resolveSasl(env);
+ const ssl = resolveSsl(env, sasl !== undefined, readFile);
+ return { ssl, sasl };
+};
+
+/** Log-safe summary: never includes credential values. */
+export const describeKafkaSecurity = (
+ config: KafkaSecurityConfig
+): { ssl: boolean; sasl: KafkaSaslMechanism | null } => ({
+ ssl: config.ssl !== false,
+ sasl: config.sasl ? (config.sasl.mechanism as KafkaSaslMechanism) : null,
+});
diff --git a/packages/queue/src/kafka.integration.test.ts b/packages/queue/src/kafka.integration.test.ts
new file mode 100644
index 000000000..1b55e57d9
--- /dev/null
+++ b/packages/queue/src/kafka.integration.test.ts
@@ -0,0 +1,267 @@
+import { connect } from 'node:net';
+import path from 'node:path';
+import { afterEach, describe, expect, it, vi } from 'vitest';
+
+// Round-trips a message through the real producer/consumer in kafka.ts against
+// a broker that requires SASL, in every supported TLS/SASL mode.
+//
+// Needs the opt-in `op-rp-auth` compose service (docker/redpanda/auth-entrypoint.sh):
+// pnpm dock:kafka-auth
+// It exposes a plaintext and a TLS SASL listener, writes its self-signed CA to
+// docker/data/op-rp-auth-certs and creates the users below. When the broker is
+// not running this file is skipped.
+const PLAIN_BROKER = 'localhost:19093';
+const TLS_BROKER = 'localhost:19094';
+const CA_PATH = path.resolve(
+ __dirname,
+ '../../../docker/data/op-rp-auth-certs/ca.crt'
+);
+const USER_512 = { username: 'op512', password: 'op512-secret' };
+const USER_256 = { username: 'op256', password: 'op256-secret' };
+
+const isBrokerUp = (): Promise<boolean> =>
+ new Promise((resolve) => {
+ const [host, port] = PLAIN_BROKER.split(':');
+ const socket = connect({ host, port: Number(port), timeout: 1000 });
+ const done = (up: boolean) => {
+ socket.destroy();
+ resolve(up);
+ };
+ socket.once('connect', () => done(true));
+ socket.once('error', () => done(false));
+ socket.once('timeout', () => done(false));
+ });
+
+const brokerUp = await isBrokerUp();
+if (!brokerUp) {
+ console.warn(
+ 'kafka.integration.test.ts skipped: op-rp-auth is not running (pnpm dock:kafka-auth)'
+ );
+}
+
+const ROUND_TRIP_TIMEOUT_MS = 30_000;
+
+const KAFKA_ENV_KEYS = [
+ 'KAFKA_BROKERS',
+ 'KAFKA_EVENTS_TOPIC',
+ 'KAFKA_CONSUMER_GROUP',
+ 'KAFKA_SSL',
+ 'KAFKA_SSL_CA_PATH',
+ 'KAFKA_SSL_REJECT_UNAUTHORIZED',
+ 'KAFKA_SASL_USERNAME',
+ 'KAFKA_SASL_PASSWORD',
+ 'KAFKA_SASL_MECHANISM',
+] as const;
+
+type KafkaModule = typeof import('./kafka');
+
+let active: KafkaModule | null = null;
+
+const loadKafka = async (
+ env: Partial<Record<(typeof KAFKA_ENV_KEYS)[number], string>>
+): Promise<KafkaModule> => {
+ vi.resetModules();
+ for (const key of KAFKA_ENV_KEYS) {
+ vi.stubEnv(key, env[key] ?? '');
+ if (env[key] === undefined) {
+ // stubEnv('') leaves an empty string, which our parser treats as unset.
+ delete process.env[key];
+ }
+ }
+ const mod = await import('./kafka');
+ active = mod;
+ return mod;
+};
+
+const uniqueTopic = () =>
+ `op-sasl-test-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`;
+
+// Produce one event, then consume it back from the beginning of the topic.
+// Returns both so the caller can assert they match.
+const roundTrip = async (
+ mod: KafkaModule
+): Promise<{ sent: Record<string, unknown>; received: unknown }> => {
+ const payload = { marker: `marker-${Date.now()}` };
+ await mod.produceIncomingEvent(
+ payload as unknown as Parameters<typeof mod.produceIncomingEvent>[0],
+ 'partition-key'
+ );
+
+ const consumer = mod.createKafkaEventsConsumer({
+ groupId: `${mod.KAFKA_CONSUMER_GROUP}-${Math.random().toString(36).slice(2, 8)}`,
+ });
+ await consumer.connect();
+ await consumer.subscribe({
+ topic: mod.KAFKA_EVENTS_TOPIC,
+ fromBeginning: true,
+ });
+
+ const received = new Promise<Record<string, unknown>>((resolve, reject) => {
+ const timer = setTimeout(
+ () => reject(new Error('timed out waiting for message')),
+ ROUND_TRIP_TIMEOUT_MS
+ );
+ consumer
+ .run({
+ eachMessage: async ({ message }) => {
+ clearTimeout(timer);
+ resolve(JSON.parse(message.value?.toString() ?? '{}'));
+ },
+ })
+ .catch(reject);
+ });
+
+ return { sent: payload, received: await received };
+};
+
+afterEach(async () => {
+ if (active) {
+ await active.disconnectKafka();
+ active = null;
+ }
+ vi.unstubAllEnvs();
+});
+
+describe.skipIf(!brokerUp)('kafka TLS/SASL integration', () => {
+ it(
+ 'SASL/SCRAM-SHA-512 over plaintext (KAFKA_SSL=false)',
+ async () => {
+ const mod = await loadKafka({
+ KAFKA_BROKERS: PLAIN_BROKER,
+ KAFKA_EVENTS_TOPIC: uniqueTopic(),
+ KAFKA_SSL: 'false',
+ KAFKA_SASL_USERNAME: USER_512.username,
+ KAFKA_SASL_PASSWORD: USER_512.password,
+ });
+ const { sent, received } = await roundTrip(mod);
+ expect(received).toEqual(sent);
+ },
+ ROUND_TRIP_TIMEOUT_MS + 5000
+ );
+
+ it(
+ 'SASL/SCRAM-SHA-512 over TLS is the default when credentials are set',
+ async () => {
+ const mod = await loadKafka({
+ KAFKA_BROKERS: TLS_BROKER,
+ KAFKA_EVENTS_TOPIC: uniqueTopic(),
+ KAFKA_SSL_CA_PATH: CA_PATH,
+ KAFKA_SASL_USERNAME: USER_512.username,
+ KAFKA_SASL_PASSWORD: USER_512.password,
+ });
+ const { sent, received } = await roundTrip(mod);
+ expect(received).toEqual(sent);
+ },
+ ROUND_TRIP_TIMEOUT_MS + 5000
+ );
+
+ it(
+ 'SASL/SCRAM-SHA-256 over TLS',
+ async () => {
+ const mod = await loadKafka({
+ KAFKA_BROKERS: TLS_BROKER,
+ KAFKA_EVENTS_TOPIC: uniqueTopic(),
+ KAFKA_SSL: 'true',
+ KAFKA_SSL_CA_PATH: CA_PATH,
+ KAFKA_SASL_MECHANISM: 'scram-sha-256',
+ KAFKA_SASL_USERNAME: USER_256.username,
+ KAFKA_SASL_PASSWORD: USER_256.password,
+ });
+ const { sent, received } = await roundTrip(mod);
+ expect(received).toEqual(sent);
+ },
+ ROUND_TRIP_TIMEOUT_MS + 5000
+ );
+
+ it(
+ 'SASL/PLAIN over TLS',
+ async () => {
+ const mod = await loadKafka({
+ KAFKA_BROKERS: TLS_BROKER,
+ KAFKA_EVENTS_TOPIC: uniqueTopic(),
+ KAFKA_SSL_CA_PATH: CA_PATH,
+ KAFKA_SASL_MECHANISM: 'plain',
+ KAFKA_SASL_USERNAME: USER_512.username,
+ KAFKA_SASL_PASSWORD: USER_512.password,
+ });
+ const { sent, received } = await roundTrip(mod);
+ expect(received).toEqual(sent);
+ },
+ ROUND_TRIP_TIMEOUT_MS + 5000
+ );
+
+ it(
+ 'TLS with an untrusted (self-signed) CA is rejected by default',
+ async () => {
+ const mod = await loadKafka({
+ KAFKA_BROKERS: TLS_BROKER,
+ KAFKA_EVENTS_TOPIC: uniqueTopic(),
+ KAFKA_SASL_USERNAME: USER_512.username,
+ KAFKA_SASL_PASSWORD: USER_512.password,
+ });
+ await expect(mod.produceIncomingEvent({} as never, 'k')).rejects.toThrow(
+ /self[- ]signed|certificate|unable to verify/i
+ );
+ },
+ ROUND_TRIP_TIMEOUT_MS + 5000
+ );
+
+ it(
+ 'TLS with an untrusted CA works with KAFKA_SSL_REJECT_UNAUTHORIZED=false',
+ async () => {
+ const mod = await loadKafka({
+ KAFKA_BROKERS: TLS_BROKER,
+ KAFKA_EVENTS_TOPIC: uniqueTopic(),
+ KAFKA_SSL_REJECT_UNAUTHORIZED: 'false',
+ KAFKA_SASL_USERNAME: USER_512.username,
+ KAFKA_SASL_PASSWORD: USER_512.password,
+ });
+ const { sent, received } = await roundTrip(mod);
+ expect(received).toEqual(sent);
+ },
+ ROUND_TRIP_TIMEOUT_MS + 5000
+ );
+
+ it(
+ 'wrong SASL password fails authentication',
+ async () => {
+ const mod = await loadKafka({
+ KAFKA_BROKERS: TLS_BROKER,
+ KAFKA_EVENTS_TOPIC: uniqueTopic(),
+ KAFKA_SSL_CA_PATH: CA_PATH,
+ KAFKA_SASL_USERNAME: USER_512.username,
+ KAFKA_SASL_PASSWORD: 'definitely-wrong',
+ });
+ await expect(mod.produceIncomingEvent({} as never, 'k')).rejects.toThrow(
+ /SASL|authentication/i
+ );
+ },
+ ROUND_TRIP_TIMEOUT_MS + 5000
+ );
+
+ it(
+ 'unauthenticated client is rejected by a SASL-only broker',
+ async () => {
+ const mod = await loadKafka({
+ KAFKA_BROKERS: TLS_BROKER,
+ KAFKA_EVENTS_TOPIC: uniqueTopic(),
+ KAFKA_SSL: 'true',
+ KAFKA_SSL_CA_PATH: CA_PATH,
+ });
+ await expect(
+ mod.produceIncomingEvent({} as never, 'k')
+ ).rejects.toThrow();
+ },
+ ROUND_TRIP_TIMEOUT_MS + 5000
+ );
+
+ it('partial SASL config fails at module load', async () => {
+ await expect(
+ loadKafka({
+ KAFKA_BROKERS: PLAIN_BROKER,
+ KAFKA_SASL_USERNAME: USER_512.username,
+ })
+ ).rejects.toThrow(/KAFKA_SASL_PASSWORD is missing/);
+ active = null;
+ });
+});
diff --git a/packages/queue/src/kafka.ts b/packages/queue/src/kafka.ts
index f0a0c5870..7c820a961 100644
--- a/packages/queue/src/kafka.ts
+++ b/packages/queue/src/kafka.ts
@@ -1,5 +1,6 @@
import { createLogger } from '@openpanel/logger';
import { type Consumer, Kafka, logLevel, type Producer } from 'kafkajs';
+import { describeKafkaSecurity, resolveKafkaSecurity } from './kafka-security';
import type { EventsQueuePayloadIncomingEvent } from './queues';
export type { KafkaMessage } from 'kafkajs';
@@ -80,6 +81,12 @@ const KAFKA_MAX_BYTES_PER_PARTITION =
export const isKafkaConfigured = (): boolean => KAFKA_BROKERS.length > 0;
+// Resolved at module load so a partial/invalid TLS+SASL config fails the
+// process at startup rather than on the first produce/consume.
+const KAFKA_SECURITY = isKafkaConfigured()
+ ? resolveKafkaSecurity(process.env)
+ : { ssl: false, sasl: undefined };
+
export const shouldUseKafka = (): boolean => isKafkaConfigured();
let kafka: Kafka | null = null;
@@ -96,6 +103,8 @@ const getKafka = (): Kafka => {
logLevel: logLevel.WARN,
requestTimeout: KAFKA_REQUEST_TIMEOUT_MS,
connectionTimeout: KAFKA_CONNECTION_TIMEOUT_MS,
+ ssl: KAFKA_SECURITY.ssl,
+ sasl: KAFKA_SECURITY.sasl,
});
}
return kafka;
@@ -129,7 +138,11 @@ const getProducer = async (): Promise<Producer> => {
.then(() => {
producer = p;
kafkaLogger.info(
- { brokers: KAFKA_BROKERS, topic: KAFKA_EVENTS_TOPIC },
+ {
+ brokers: KAFKA_BROKERS,
+ topic: KAFKA_EVENTS_TOPIC,
+ ...describeKafkaSecurity(KAFKA_SECURITY),
+ },
'kafka producer connected'
);
return p;
diff --git a/packages/queue/vitest.config.ts b/packages/queue/vitest.config.ts
new file mode 100644
index 000000000..f87a2039f
--- /dev/null
+++ b/packages/queue/vitest.config.ts
@@ -0,0 +1,3 @@
+import { getSharedVitestConfig } from '../../vitest.shared';
+
+export default getSharedVitestConfig({ __dirname });