OpenPanel is an open-source web and product analytics platform, an open-source alternative to Mixpanel with optional self-hosting.
feat(kafka): support TLS and SASL auth for external brokers
Carl-Gerhard Lindesvärdf80b12411269
diff --git a/apps/public/content/docs/self-hosting/environment-variables.mdx b/apps/public/content/docs/self-hosting/environment-variables.mdxindex bb4442ebf..9b1d3fc5e 100644--- a/apps/public/content/docs/self-hosting/environment-variables.mdx+++ b/apps/public/content/docs/self-hosting/environment-variables.mdx@@ -742,6 +742,8 @@ OpenPanel can optionally ingest events through a Kafka-compatible broker (KafkaRouting is all-or-nothing: if `KAFKA_BROKERS` is set, every event is produced to the Kafka topic; otherwise all events stay on the default Redis-backed `events` queue. Set `ENABLED_QUEUES=events_kafka` on the worker(s) that should consume from Kafka.+Connections are unauthenticated plaintext by default. For managed or production clusters, enable TLS with `KAFKA_SSL` and SASL authentication (`plain`, `scram-sha-256` or `scram-sha-512`) with the `KAFKA_SASL_*` variables below.+<Callout>The client speaks the Kafka wire protocol via [kafkajs](https://kafka.js.org/), so any Kafka-compatible broker works. The bundled `docker-compose.yml` uses [Redpanda](https://redpanda.com/) as a lightweight, single-binary option, but you can point `KAFKA_BROKERS` at any Kafka cluster.</Callout>@@ -761,6 +763,98 @@ KAFKA_BROKERS=redpanda:9092KAFKA_BROKERS=kafka-1:9092,kafka-2:9092,kafka-3:9092```+### KAFKA_SSL++**Type**: `boolean`+**Required**: No+**Default**: `false` (automatically `true` when SASL credentials are set)++Connect to the brokers over TLS. Set this to `true` for a TLS-only cluster without authentication. When `KAFKA_SASL_USERNAME`/`KAFKA_SASL_PASSWORD` are set, TLS is enabled automatically; set `KAFKA_SSL=false` explicitly to send SASL over plaintext (only sensible on a trusted internal network).++**Example**:+```bash+KAFKA_SSL=true+```++### KAFKA_SSL_CA_PATH++**Type**: `string`+**Required**: No+**Default**: None (system trust store)++Path to a PEM-encoded CA certificate (or bundle) to trust in addition to the system trust store. Use this when your brokers present certificates signed by a private or self-signed CA. Requires TLS to be enabled.++**Example**:+```bash+KAFKA_SSL_CA_PATH=/etc/openpanel/kafka-ca.pem+```++### KAFKA_SSL_REJECT_UNAUTHORIZED++**Type**: `boolean`+**Required**: No+**Default**: `true`++Whether to reject broker certificates that cannot be verified. Only set to `false` for local development against a self-signed broker; prefer `KAFKA_SSL_CA_PATH` in production. Requires TLS to be enabled.++**Example**:+```bash+KAFKA_SSL_REJECT_UNAUTHORIZED=false+```++### KAFKA_SASL_USERNAME++**Type**: `string`+**Required**: No (required together with `KAFKA_SASL_PASSWORD` to enable SASL)+**Default**: None++SASL username. Setting a username without a password (or vice versa) is a configuration error and the process refuses to start. Credential values are never logged.++**Example**:+```bash+KAFKA_SASL_USERNAME=openpanel+```++### KAFKA_SASL_PASSWORD++**Type**: `string`+**Required**: No (required together with `KAFKA_SASL_USERNAME` to enable SASL)+**Default**: None++SASL password.++**Example**:+```bash+KAFKA_SASL_PASSWORD=your-secret+```++### KAFKA_SASL_MECHANISM++**Type**: `string`+**Required**: No+**Default**: `scram-sha-512`++SASL mechanism to authenticate with. One of `plain`, `scram-sha-256` or `scram-sha-512` (case-insensitive). Any other value is rejected at startup, before a Kafka client is created.++**Example**:+```bash+# TLS + SASL/SCRAM-SHA-512 (the default when credentials are set)+KAFKA_BROKERS=kafka-1.example.com:9093,kafka-2.example.com:9093+KAFKA_SASL_USERNAME=openpanel+KAFKA_SASL_PASSWORD=your-secret++# TLS + SASL/PLAIN with a private CA+KAFKA_BROKERS=kafka.internal:9093+KAFKA_SASL_MECHANISM=plain+KAFKA_SASL_USERNAME=openpanel+KAFKA_SASL_PASSWORD=your-secret+KAFKA_SSL_CA_PATH=/etc/openpanel/kafka-ca.pem+```++<Callout>+The producer runs in idempotent mode, so the SASL principal needs the `IdempotentWrite` cluster permission in addition to `Write`/`Describe` on the topic and `Read` on the consumer group. Managed Kafka providers usually grant this by default; on a locked-down cluster you may need to add it explicitly.+</Callout>+### KAFKA_EVENTS_TOPIC**Type**: `string`diff --git a/docker-compose.yml b/docker-compose.ymlindex 189574789..782709a01 100644--- a/docker-compose.yml+++ b/docker-compose.yml@@ -62,6 +62,25 @@ services:- "19092:19092" # Kafka API (external)- "9644:9644" # Admin API+ # Opt-in (profile "kafka-auth"): standalone broker that requires SASL, with a+ # TLS listener, for verifying KAFKA_SSL/KAFKA_SASL_* support.+ # See docker/redpanda/auth-entrypoint.sh. Not started by `pnpm dock:up`.+ op-rp-auth:+ image: redpandadata/redpanda:v25.1.4+ profiles: ["kafka-auth"]+ entrypoint: ["/auth-entrypoint.sh"]+ volumes:+ - ./docker/data/op-rp-auth-certs:/certs+ - ./docker/redpanda/auth-entrypoint.sh:/auth-entrypoint.sh:ro+ ports:+ - "19093:19093" # Kafka API, SASL required, plaintext+ - "19094:19094" # Kafka API, SASL required, TLS+ healthcheck:+ test: ["CMD-SHELL", "rpk cluster health -X admin.hosts=localhost:9644 | grep -q 'Healthy:.*true'"]+ interval: 5s+ timeout: 3s+ retries: 20+op-rp-console:image: redpandadata/console:v3.7.2restart: alwaysdiff --git a/docker/redpanda/auth-entrypoint.sh b/docker/redpanda/auth-entrypoint.shnew file mode 100755index 000000000..f9aa090b3--- /dev/null+++ b/docker/redpanda/auth-entrypoint.sh@@ -0,0 +1,60 @@+#!/usr/bin/env bash+# Standalone Redpanda that REQUIRES SASL, for verifying the TLS/SASL support in+# packages/queue against a real broker. Not part of the default stack:+#+# pnpm dock:kafka-auth # start (compose profile "kafka-auth")+# pnpm vitest run packages/queue/src/kafka.integration.test.ts+#+# Listeners (both require SASL):+# localhost:19093 plaintext+# localhost:19094 TLS, self-signed CA written to docker/data/op-rp-auth-certs+# Users: op512 (SCRAM-SHA-512), op256 (SCRAM-SHA-256); PLAIN works with both.+set -euo pipefail++CERT_DIR=/certs+SASL_PORT=19093+TLS_PORT=19094++gen_certs() {+ if [[ -f "$CERT_DIR/ca.crt" && -f "$CERT_DIR/broker.crt" && -f "$CERT_DIR/broker.key" ]]; then+ return+ fi+ echo "generating self-signed TLS certs in $CERT_DIR"+ openssl req -x509 -newkey rsa:2048 -nodes -keyout "$CERT_DIR/ca.key" -out "$CERT_DIR/ca.crt" \+ -days 3650 -subj "/CN=openpanel-dev-ca" 2>/dev/null+ openssl req -newkey rsa:2048 -nodes -keyout "$CERT_DIR/broker.key" -out "$CERT_DIR/broker.csr" \+ -subj "/CN=localhost" 2>/dev/null+ printf "subjectAltName=DNS:localhost,IP:127.0.0.1\n" > "$CERT_DIR/san.ext"+ openssl x509 -req -in "$CERT_DIR/broker.csr" -CA "$CERT_DIR/ca.crt" -CAkey "$CERT_DIR/ca.key" \+ -CAcreateserial -out "$CERT_DIR/broker.crt" -days 3650 -extfile "$CERT_DIR/san.ext" 2>/dev/null+ chmod 644 "$CERT_DIR"/*.crt "$CERT_DIR"/*.key+}++configure_cluster() {+ for _ in $(seq 1 60); do+ if rpk cluster health -X admin.hosts=localhost:9644 >/dev/null 2>&1; then+ break+ fi+ sleep 1+ done+ rpk cluster config set enable_sasl true -X admin.hosts=localhost:9644+ rpk cluster config set sasl_mechanisms '["SCRAM","PLAIN"]' -X admin.hosts=localhost:9644+ rpk cluster config set superusers '["op512","op256"]' -X admin.hosts=localhost:9644+ # Redpanda stores one SCRAM mechanism per user; PLAIN works against either.+ rpk security user create op512 -p op512-secret --mechanism SCRAM-SHA-512 -X admin.hosts=localhost:9644 || true+ rpk security user create op256 -p op256-secret --mechanism SCRAM-SHA-256 -X admin.hosts=localhost:9644 || true+ echo "kafka-auth broker ready"+}++gen_certs+configure_cluster &++exec rpk redpanda start \+ --smp=1 \+ --memory=512M \+ --overprovisioned \+ --mode=dev-container \+ --default-log-level=warn \+ --kafka-addr="sasl://0.0.0.0:$SASL_PORT,tls://0.0.0.0:$TLS_PORT" \+ --advertise-kafka-addr="sasl://localhost:$SASL_PORT,tls://localhost:$TLS_PORT" \+ --set "redpanda.kafka_api_tls=[{\"name\":\"tls\",\"enabled\":true,\"cert_file\":\"$CERT_DIR/broker.crt\",\"key_file\":\"$CERT_DIR/broker.key\",\"truststore_file\":\"$CERT_DIR/ca.crt\",\"require_client_auth\":false}]"diff --git a/package.json b/package.jsonindex 8d3681acf..5978fe7dc 100644--- a/package.json+++ b/package.json@@ -12,6 +12,7 @@"gen:referrers": "pnpm -r --filter common gen:referrers","dock:up": "docker compose up -d","dock:down": "docker compose down",+ "dock:kafka-auth": "docker compose --profile kafka-auth up -d --wait op-rp-auth","dock:ch": "docker compose exec -it op-ch clickhouse-client -d openpanel","dock:redis": "docker compose exec -it op-kv redis-cli","db:codegen": "pnpm -r --filter db run codegen",diff --git a/packages/queue/package.json b/packages/queue/package.jsonindex 989f30f48..33df9e692 100644--- a/packages/queue/package.json+++ b/packages/queue/package.json@@ -4,7 +4,8 @@"type": "module","main": "index.ts","scripts": {- "typecheck": "tsc --noEmit"+ "typecheck": "tsc --noEmit",+ "test": "vitest run"},"dependencies": {"@openpanel/db": "workspace:*",diff --git a/packages/queue/src/kafka-security.test.ts b/packages/queue/src/kafka-security.test.tsnew file mode 100644index 000000000..d6ef92592--- /dev/null+++ b/packages/queue/src/kafka-security.test.ts@@ -0,0 +1,180 @@+import { describe, expect, it } from 'vitest';+import { describeKafkaSecurity, resolveKafkaSecurity } from './kafka-security';++const readFile = (path: string): string => {+ if (path === '/certs/ca.pem') {+ return '-----BEGIN CERTIFICATE-----\nfake\n-----END CERTIFICATE-----';+ }+ throw new Error(`ENOENT: no such file or directory, open '${path}'`);+};++describe('resolveKafkaSecurity', () => {+ it('defaults to plaintext without auth when nothing is configured', () => {+ expect(resolveKafkaSecurity({}, readFile)).toEqual({+ ssl: false,+ sasl: undefined,+ });+ });++ it('enables TLS only with KAFKA_SSL=true', () => {+ expect(resolveKafkaSecurity({ KAFKA_SSL: 'true' }, readFile)).toEqual({+ ssl: true,+ sasl: undefined,+ });+ expect(resolveKafkaSecurity({ KAFKA_SSL: '1' }, readFile).ssl).toBe(true);+ });++ it('passes a custom CA and rejectUnauthorized through to tls options', () => {+ expect(+ resolveKafkaSecurity(+ {+ KAFKA_SSL: 'true',+ KAFKA_SSL_CA_PATH: '/certs/ca.pem',+ KAFKA_SSL_REJECT_UNAUTHORIZED: 'false',+ },+ readFile+ ).ssl+ ).toEqual({+ ca: ['-----BEGIN CERTIFICATE-----\nfake\n-----END CERTIFICATE-----'],+ rejectUnauthorized: false,+ });+ });++ it('fails with a clear error when the CA file cannot be read', () => {+ expect(() =>+ resolveKafkaSecurity(+ { KAFKA_SSL: 'true', KAFKA_SSL_CA_PATH: '/missing.pem' },+ readFile+ )+ ).toThrow(/Failed to read KAFKA_SSL_CA_PATH "\/missing.pem"/);+ });++ it('rejects TLS-only options when TLS is disabled', () => {+ expect(() =>+ resolveKafkaSecurity({ KAFKA_SSL_CA_PATH: '/certs/ca.pem' }, readFile)+ ).toThrow(/require TLS/);+ expect(() =>+ resolveKafkaSecurity(+ { KAFKA_SSL: 'false', KAFKA_SSL_REJECT_UNAUTHORIZED: 'false' },+ readFile+ )+ ).toThrow(/require TLS/);+ });++ it('rejects non-boolean KAFKA_SSL values', () => {+ expect(() => resolveKafkaSecurity({ KAFKA_SSL: 'yes' }, readFile)).toThrow(+ /KAFKA_SSL must be "true" or "false"/+ );+ });++ it('defaults SASL to scram-sha-512 over TLS when credentials are present', () => {+ expect(+ resolveKafkaSecurity(+ { KAFKA_SASL_USERNAME: 'op', KAFKA_SASL_PASSWORD: 'secret' },+ readFile+ )+ ).toEqual({+ ssl: true,+ sasl: { mechanism: 'scram-sha-512', username: 'op', password: 'secret' },+ });+ });++ it.each([+ 'plain',+ 'scram-sha-256',+ 'scram-sha-512',+ ] as const)('supports mechanism %s (case-insensitive)', (mechanism) => {+ const config = resolveKafkaSecurity(+ {+ KAFKA_SASL_USERNAME: 'op',+ KAFKA_SASL_PASSWORD: 'secret',+ KAFKA_SASL_MECHANISM: mechanism.toUpperCase(),+ },+ readFile+ );+ expect(config.sasl?.mechanism).toBe(mechanism);+ expect(config.ssl).toBe(true);+ });++ it('allows SASL over plaintext when KAFKA_SSL=false is explicit', () => {+ expect(+ resolveKafkaSecurity(+ {+ KAFKA_SSL: 'false',+ KAFKA_SASL_USERNAME: 'op',+ KAFKA_SASL_PASSWORD: 'secret',+ KAFKA_SASL_MECHANISM: 'plain',+ },+ readFile+ )+ ).toEqual({+ ssl: false,+ sasl: { mechanism: 'plain', username: 'op', password: 'secret' },+ });+ });++ it('rejects unsupported mechanisms before creating a client', () => {+ expect(() =>+ resolveKafkaSecurity(+ {+ KAFKA_SASL_USERNAME: 'op',+ KAFKA_SASL_PASSWORD: 'secret',+ KAFKA_SASL_MECHANISM: 'oauthbearer',+ },+ readFile+ )+ ).toThrow(+ 'Unsupported KAFKA_SASL_MECHANISM "oauthbearer". Supported: plain, scram-sha-256, scram-sha-512'+ );+ });++ it('fails fast when only one of username/password is set', () => {+ expect(() =>+ resolveKafkaSecurity({ KAFKA_SASL_USERNAME: 'op' }, readFile)+ ).toThrow(/KAFKA_SASL_PASSWORD is missing/);+ expect(() =>+ resolveKafkaSecurity({ KAFKA_SASL_PASSWORD: 'secret' }, readFile)+ ).toThrow(/KAFKA_SASL_USERNAME is missing/);+ });++ it('fails fast when a mechanism is set without credentials', () => {+ expect(() =>+ resolveKafkaSecurity({ KAFKA_SASL_MECHANISM: 'plain' }, readFile)+ ).toThrow(/KAFKA_SASL_MECHANISM is set but/);+ });++ it('never leaks credentials in error messages', () => {+ const env = {+ KAFKA_SASL_USERNAME: 'op',+ KAFKA_SASL_PASSWORD: 'hunter2-secret',+ KAFKA_SASL_MECHANISM: 'nope',+ };+ let message = '';+ try {+ resolveKafkaSecurity(env, readFile);+ } catch (err) {+ message = err instanceof Error ? err.message : String(err);+ }+ expect(message).not.toContain('hunter2-secret');+ });+});++describe('describeKafkaSecurity', () => {+ it('summarises without credential values', () => {+ const config = resolveKafkaSecurity(+ { KAFKA_SASL_USERNAME: 'op', KAFKA_SASL_PASSWORD: 'secret' },+ readFile+ );+ expect(describeKafkaSecurity(config)).toEqual({+ ssl: true,+ sasl: 'scram-sha-512',+ });+ expect(JSON.stringify(describeKafkaSecurity(config))).not.toContain(+ 'secret'+ );+ expect(describeKafkaSecurity(resolveKafkaSecurity({}, readFile))).toEqual({+ ssl: false,+ sasl: null,+ });+ });+});diff --git a/packages/queue/src/kafka-security.ts b/packages/queue/src/kafka-security.tsnew file mode 100644index 000000000..a83cd5ce6--- /dev/null+++ b/packages/queue/src/kafka-security.ts@@ -0,0 +1,150 @@+import { readFileSync } from 'node:fs';+import type { ConnectionOptions } from 'node:tls';+import type { SASLOptions } from 'kafkajs';++// Env-driven TLS/SASL config for external Kafka brokers. Kept free of any+// project imports so it can be unit-tested in isolation.+//+// Supported modes (all opt-in via env, unauthenticated plaintext by default):+// - TLS only KAFKA_SSL=true+// - SASL/PLAIN over TLS KAFKA_SASL_MECHANISM=plain + credentials+// - SASL/SCRAM-SHA-256 over TLS KAFKA_SASL_MECHANISM=scram-sha-256 + credentials+// - SASL/SCRAM-SHA-512 over TLS KAFKA_SASL_MECHANISM=scram-sha-512 + credentials (default)+// TLS is implied when SASL credentials are present; set KAFKA_SSL=false to+// send SASL over plaintext (dev/internal networks only).++export const KAFKA_SASL_MECHANISMS = [+ 'plain',+ 'scram-sha-256',+ 'scram-sha-512',+] as const;+export type KafkaSaslMechanism = (typeof KAFKA_SASL_MECHANISMS)[number];+const DEFAULT_SASL_MECHANISM: KafkaSaslMechanism = 'scram-sha-512';++// Shape-compatible with process.env. Keys read:+// KAFKA_SSL, KAFKA_SSL_CA_PATH, KAFKA_SSL_REJECT_UNAUTHORIZED,+// KAFKA_SASL_USERNAME, KAFKA_SASL_PASSWORD, KAFKA_SASL_MECHANISM+export type KafkaSecurityEnv = Readonly<Record<string, string | undefined>>;++export type KafkaSecurityConfig = {+ ssl: ConnectionOptions | boolean;+ sasl: SASLOptions | undefined;+};++const isSaslMechanism = (value: string): value is KafkaSaslMechanism =>+ (KAFKA_SASL_MECHANISMS as readonly string[]).includes(value);++const parseBoolean = (+ name: string,+ raw: string | undefined+): boolean | undefined => {+ if (raw === undefined || raw.trim() === '') {+ return undefined;+ }+ const value = raw.trim().toLowerCase();+ if (value === 'true' || value === '1') {+ return true;+ }+ if (value === 'false' || value === '0') {+ return false;+ }+ throw new Error(`${name} must be "true" or "false" (got "${raw}")`);+};++const resolveSasl = (env: KafkaSecurityEnv): SASLOptions | undefined => {+ const username = env.KAFKA_SASL_USERNAME;+ const password = env.KAFKA_SASL_PASSWORD;+ const hasUsername = Boolean(username);+ const hasPassword = Boolean(password);++ if (!(hasUsername || hasPassword)) {+ if (env.KAFKA_SASL_MECHANISM) {+ throw new Error(+ 'KAFKA_SASL_MECHANISM is set but KAFKA_SASL_USERNAME and KAFKA_SASL_PASSWORD are missing'+ );+ }+ return undefined;+ }+ if (!(hasUsername && hasPassword)) {+ const missing = hasUsername ? 'KAFKA_SASL_PASSWORD' : 'KAFKA_SASL_USERNAME';+ throw new Error(+ `Kafka SASL is partially configured: ${missing} is missing (both KAFKA_SASL_USERNAME and KAFKA_SASL_PASSWORD are required)`+ );+ }++ const rawMechanism = env.KAFKA_SASL_MECHANISM?.trim().toLowerCase();+ const mechanism = rawMechanism || DEFAULT_SASL_MECHANISM;+ if (!isSaslMechanism(mechanism)) {+ throw new Error(+ `Unsupported KAFKA_SASL_MECHANISM "${env.KAFKA_SASL_MECHANISM}". Supported: ${KAFKA_SASL_MECHANISMS.join(', ')}`+ );+ }++ return {+ mechanism,+ username: username as string,+ password: password as string,+ };+};++const resolveSsl = (+ env: KafkaSecurityEnv,+ saslEnabled: boolean,+ readFile: (path: string) => string+): ConnectionOptions | boolean => {+ // SASL credentials go over TLS unless explicitly opted out.+ const enabled = parseBoolean('KAFKA_SSL', env.KAFKA_SSL) ?? saslEnabled;+ const caPath = env.KAFKA_SSL_CA_PATH?.trim();+ const rejectUnauthorized = parseBoolean(+ 'KAFKA_SSL_REJECT_UNAUTHORIZED',+ env.KAFKA_SSL_REJECT_UNAUTHORIZED+ );++ if (!enabled) {+ if (caPath || rejectUnauthorized !== undefined) {+ throw new Error(+ 'KAFKA_SSL_CA_PATH / KAFKA_SSL_REJECT_UNAUTHORIZED require TLS; set KAFKA_SSL=true'+ );+ }+ return false;+ }++ const options: ConnectionOptions = {};+ if (caPath) {+ try {+ options.ca = [readFile(caPath)];+ } catch (err) {+ throw new Error(+ `Failed to read KAFKA_SSL_CA_PATH "${caPath}": ${err instanceof Error ? err.message : String(err)}`+ );+ }+ }+ if (rejectUnauthorized !== undefined) {+ options.rejectUnauthorized = rejectUnauthorized;+ }+ return Object.keys(options).length > 0 ? options : true;+};++const readPemFile = (path: string): string => readFileSync(path, 'utf8');++/**+ * Build the kafkajs `ssl` / `sasl` options from env. Throws a descriptive+ * Error on inconsistent configuration so misconfiguration surfaces at+ * startup instead of as an opaque broker handshake failure.+ */+export const resolveKafkaSecurity = (+ env: KafkaSecurityEnv,+ readFile: (path: string) => string = readPemFile+): KafkaSecurityConfig => {+ const sasl = resolveSasl(env);+ const ssl = resolveSsl(env, sasl !== undefined, readFile);+ return { ssl, sasl };+};++/** Log-safe summary: never includes credential values. */+export const describeKafkaSecurity = (+ config: KafkaSecurityConfig+): { ssl: boolean; sasl: KafkaSaslMechanism | null } => ({+ ssl: config.ssl !== false,+ sasl: config.sasl ? (config.sasl.mechanism as KafkaSaslMechanism) : null,+});diff --git a/packages/queue/src/kafka.integration.test.ts b/packages/queue/src/kafka.integration.test.tsnew file mode 100644index 000000000..1b55e57d9--- /dev/null+++ b/packages/queue/src/kafka.integration.test.ts@@ -0,0 +1,267 @@+import { connect } from 'node:net';+import path from 'node:path';+import { afterEach, describe, expect, it, vi } from 'vitest';++// Round-trips a message through the real producer/consumer in kafka.ts against+// a broker that requires SASL, in every supported TLS/SASL mode.+//+// Needs the opt-in `op-rp-auth` compose service (docker/redpanda/auth-entrypoint.sh):+// pnpm dock:kafka-auth+// It exposes a plaintext and a TLS SASL listener, writes its self-signed CA to+// docker/data/op-rp-auth-certs and creates the users below. When the broker is+// not running this file is skipped.+const PLAIN_BROKER = 'localhost:19093';+const TLS_BROKER = 'localhost:19094';+const CA_PATH = path.resolve(+ __dirname,+ '../../../docker/data/op-rp-auth-certs/ca.crt'+);+const USER_512 = { username: 'op512', password: 'op512-secret' };+const USER_256 = { username: 'op256', password: 'op256-secret' };++const isBrokerUp = (): Promise<boolean> =>+ new Promise((resolve) => {+ const [host, port] = PLAIN_BROKER.split(':');+ const socket = connect({ host, port: Number(port), timeout: 1000 });+ const done = (up: boolean) => {+ socket.destroy();+ resolve(up);+ };+ socket.once('connect', () => done(true));+ socket.once('error', () => done(false));+ socket.once('timeout', () => done(false));+ });++const brokerUp = await isBrokerUp();+if (!brokerUp) {+ console.warn(+ 'kafka.integration.test.ts skipped: op-rp-auth is not running (pnpm dock:kafka-auth)'+ );+}++const ROUND_TRIP_TIMEOUT_MS = 30_000;++const KAFKA_ENV_KEYS = [+ 'KAFKA_BROKERS',+ 'KAFKA_EVENTS_TOPIC',+ 'KAFKA_CONSUMER_GROUP',+ 'KAFKA_SSL',+ 'KAFKA_SSL_CA_PATH',+ 'KAFKA_SSL_REJECT_UNAUTHORIZED',+ 'KAFKA_SASL_USERNAME',+ 'KAFKA_SASL_PASSWORD',+ 'KAFKA_SASL_MECHANISM',+] as const;++type KafkaModule = typeof import('./kafka');++let active: KafkaModule | null = null;++const loadKafka = async (+ env: Partial<Record<(typeof KAFKA_ENV_KEYS)[number], string>>+): Promise<KafkaModule> => {+ vi.resetModules();+ for (const key of KAFKA_ENV_KEYS) {+ vi.stubEnv(key, env[key] ?? '');+ if (env[key] === undefined) {+ // stubEnv('') leaves an empty string, which our parser treats as unset.+ delete process.env[key];+ }+ }+ const mod = await import('./kafka');+ active = mod;+ return mod;+};++const uniqueTopic = () =>+ `op-sasl-test-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`;++// Produce one event, then consume it back from the beginning of the topic.+// Returns both so the caller can assert they match.+const roundTrip = async (+ mod: KafkaModule+): Promise<{ sent: Record<string, unknown>; received: unknown }> => {+ const payload = { marker: `marker-${Date.now()}` };+ await mod.produceIncomingEvent(+ payload as unknown as Parameters<typeof mod.produceIncomingEvent>[0],+ 'partition-key'+ );++ const consumer = mod.createKafkaEventsConsumer({+ groupId: `${mod.KAFKA_CONSUMER_GROUP}-${Math.random().toString(36).slice(2, 8)}`,+ });+ await consumer.connect();+ await consumer.subscribe({+ topic: mod.KAFKA_EVENTS_TOPIC,+ fromBeginning: true,+ });++ const received = new Promise<Record<string, unknown>>((resolve, reject) => {+ const timer = setTimeout(+ () => reject(new Error('timed out waiting for message')),+ ROUND_TRIP_TIMEOUT_MS+ );+ consumer+ .run({+ eachMessage: async ({ message }) => {+ clearTimeout(timer);+ resolve(JSON.parse(message.value?.toString() ?? '{}'));+ },+ })+ .catch(reject);+ });++ return { sent: payload, received: await received };+};++afterEach(async () => {+ if (active) {+ await active.disconnectKafka();+ active = null;+ }+ vi.unstubAllEnvs();+});++describe.skipIf(!brokerUp)('kafka TLS/SASL integration', () => {+ it(+ 'SASL/SCRAM-SHA-512 over plaintext (KAFKA_SSL=false)',+ async () => {+ const mod = await loadKafka({+ KAFKA_BROKERS: PLAIN_BROKER,+ KAFKA_EVENTS_TOPIC: uniqueTopic(),+ KAFKA_SSL: 'false',+ KAFKA_SASL_USERNAME: USER_512.username,+ KAFKA_SASL_PASSWORD: USER_512.password,+ });+ const { sent, received } = await roundTrip(mod);+ expect(received).toEqual(sent);+ },+ ROUND_TRIP_TIMEOUT_MS + 5000+ );++ it(+ 'SASL/SCRAM-SHA-512 over TLS is the default when credentials are set',+ async () => {+ const mod = await loadKafka({+ KAFKA_BROKERS: TLS_BROKER,+ KAFKA_EVENTS_TOPIC: uniqueTopic(),+ KAFKA_SSL_CA_PATH: CA_PATH,+ KAFKA_SASL_USERNAME: USER_512.username,+ KAFKA_SASL_PASSWORD: USER_512.password,+ });+ const { sent, received } = await roundTrip(mod);+ expect(received).toEqual(sent);+ },+ ROUND_TRIP_TIMEOUT_MS + 5000+ );++ it(+ 'SASL/SCRAM-SHA-256 over TLS',+ async () => {+ const mod = await loadKafka({+ KAFKA_BROKERS: TLS_BROKER,+ KAFKA_EVENTS_TOPIC: uniqueTopic(),+ KAFKA_SSL: 'true',+ KAFKA_SSL_CA_PATH: CA_PATH,+ KAFKA_SASL_MECHANISM: 'scram-sha-256',+ KAFKA_SASL_USERNAME: USER_256.username,+ KAFKA_SASL_PASSWORD: USER_256.password,+ });+ const { sent, received } = await roundTrip(mod);+ expect(received).toEqual(sent);+ },+ ROUND_TRIP_TIMEOUT_MS + 5000+ );++ it(+ 'SASL/PLAIN over TLS',+ async () => {+ const mod = await loadKafka({+ KAFKA_BROKERS: TLS_BROKER,+ KAFKA_EVENTS_TOPIC: uniqueTopic(),+ KAFKA_SSL_CA_PATH: CA_PATH,+ KAFKA_SASL_MECHANISM: 'plain',+ KAFKA_SASL_USERNAME: USER_512.username,+ KAFKA_SASL_PASSWORD: USER_512.password,+ });+ const { sent, received } = await roundTrip(mod);+ expect(received).toEqual(sent);+ },+ ROUND_TRIP_TIMEOUT_MS + 5000+ );++ it(+ 'TLS with an untrusted (self-signed) CA is rejected by default',+ async () => {+ const mod = await loadKafka({+ KAFKA_BROKERS: TLS_BROKER,+ KAFKA_EVENTS_TOPIC: uniqueTopic(),+ KAFKA_SASL_USERNAME: USER_512.username,+ KAFKA_SASL_PASSWORD: USER_512.password,+ });+ await expect(mod.produceIncomingEvent({} as never, 'k')).rejects.toThrow(+ /self[- ]signed|certificate|unable to verify/i+ );+ },+ ROUND_TRIP_TIMEOUT_MS + 5000+ );++ it(+ 'TLS with an untrusted CA works with KAFKA_SSL_REJECT_UNAUTHORIZED=false',+ async () => {+ const mod = await loadKafka({+ KAFKA_BROKERS: TLS_BROKER,+ KAFKA_EVENTS_TOPIC: uniqueTopic(),+ KAFKA_SSL_REJECT_UNAUTHORIZED: 'false',+ KAFKA_SASL_USERNAME: USER_512.username,+ KAFKA_SASL_PASSWORD: USER_512.password,+ });+ const { sent, received } = await roundTrip(mod);+ expect(received).toEqual(sent);+ },+ ROUND_TRIP_TIMEOUT_MS + 5000+ );++ it(+ 'wrong SASL password fails authentication',+ async () => {+ const mod = await loadKafka({+ KAFKA_BROKERS: TLS_BROKER,+ KAFKA_EVENTS_TOPIC: uniqueTopic(),+ KAFKA_SSL_CA_PATH: CA_PATH,+ KAFKA_SASL_USERNAME: USER_512.username,+ KAFKA_SASL_PASSWORD: 'definitely-wrong',+ });+ await expect(mod.produceIncomingEvent({} as never, 'k')).rejects.toThrow(+ /SASL|authentication/i+ );+ },+ ROUND_TRIP_TIMEOUT_MS + 5000+ );++ it(+ 'unauthenticated client is rejected by a SASL-only broker',+ async () => {+ const mod = await loadKafka({+ KAFKA_BROKERS: TLS_BROKER,+ KAFKA_EVENTS_TOPIC: uniqueTopic(),+ KAFKA_SSL: 'true',+ KAFKA_SSL_CA_PATH: CA_PATH,+ });+ await expect(+ mod.produceIncomingEvent({} as never, 'k')+ ).rejects.toThrow();+ },+ ROUND_TRIP_TIMEOUT_MS + 5000+ );++ it('partial SASL config fails at module load', async () => {+ await expect(+ loadKafka({+ KAFKA_BROKERS: PLAIN_BROKER,+ KAFKA_SASL_USERNAME: USER_512.username,+ })+ ).rejects.toThrow(/KAFKA_SASL_PASSWORD is missing/);+ active = null;+ });+});diff --git a/packages/queue/src/kafka.ts b/packages/queue/src/kafka.tsindex f0a0c5870..7c820a961 100644--- a/packages/queue/src/kafka.ts+++ b/packages/queue/src/kafka.ts@@ -1,5 +1,6 @@import { createLogger } from '@openpanel/logger';import { type Consumer, Kafka, logLevel, type Producer } from 'kafkajs';+import { describeKafkaSecurity, resolveKafkaSecurity } from './kafka-security';import type { EventsQueuePayloadIncomingEvent } from './queues';export type { KafkaMessage } from 'kafkajs';@@ -80,6 +81,12 @@ const KAFKA_MAX_BYTES_PER_PARTITION =export const isKafkaConfigured = (): boolean => KAFKA_BROKERS.length > 0;+// Resolved at module load so a partial/invalid TLS+SASL config fails the+// process at startup rather than on the first produce/consume.+const KAFKA_SECURITY = isKafkaConfigured()+ ? resolveKafkaSecurity(process.env)+ : { ssl: false, sasl: undefined };+export const shouldUseKafka = (): boolean => isKafkaConfigured();let kafka: Kafka | null = null;@@ -96,6 +103,8 @@ const getKafka = (): Kafka => {logLevel: logLevel.WARN,requestTimeout: KAFKA_REQUEST_TIMEOUT_MS,connectionTimeout: KAFKA_CONNECTION_TIMEOUT_MS,+ ssl: KAFKA_SECURITY.ssl,+ sasl: KAFKA_SECURITY.sasl,});}return kafka;@@ -129,7 +138,11 @@ const getProducer = async (): Promise<Producer> => {.then(() => {producer = p;kafkaLogger.info(- { brokers: KAFKA_BROKERS, topic: KAFKA_EVENTS_TOPIC },+ {+ brokers: KAFKA_BROKERS,+ topic: KAFKA_EVENTS_TOPIC,+ ...describeKafkaSecurity(KAFKA_SECURITY),+ },'kafka producer connected');return p;diff --git a/packages/queue/vitest.config.ts b/packages/queue/vitest.config.tsnew file mode 100644index 000000000..f87a2039f--- /dev/null+++ b/packages/queue/vitest.config.ts@@ -0,0 +1,3 @@+import { getSharedVitestConfig } from '../../vitest.shared';++export default getSharedVitestConfig({ __dirname });