Skip to content

Infisical is the open-source platform for securing secrets, certificates, access, and AI Agents.

feat(dynamic-secrets): support ZooKeeper Kafka clusters and multiple bootstrap servers

Dimitris Staratzis2bbe36660617
diff --git a/backend/src/ee/services/dynamic-secret/providers/kafka.ts b/backend/src/ee/services/dynamic-secret/providers/kafka.ts
index 4dbe29838a..2a4f3376a5 100644
--- a/backend/src/ee/services/dynamic-secret/providers/kafka.ts
+++ b/backend/src/ee/services/dynamic-secret/providers/kafka.ts
@@ -14,6 +14,7 @@ import {
deleteAclsV3,
describeAclsV3,
findErrorBy,
+ metadataV9,
ResourcePatternTypes,
ResourceTypes,
ScramMechanisms
@@ -32,7 +33,7 @@ import { verifyHostInputValidity } from "../dynamic-secret-fns";
import { DynamicSecretKafkaSchema, TDynamicProviderFns } from "./models";
import { generateUsername } from "./templateUtils";
-type TKafkaProviderInputs = z.infer<typeof DynamicSecretKafkaSchema> & { hostIp: string };
+type TKafkaProviderInputs = z.infer<typeof DynamicSecretKafkaSchema>;
const pbkdf2Async = promisify(pbkdf2);
@@ -91,86 +92,100 @@ const deleteKafkaUser = async (connection: Connection, username: string) => {
export const KafkaProvider = (): TDynamicProviderFns => {
const validateProviderInputs = async (inputs: unknown) => {
const providerInputs = await DynamicSecretKafkaSchema.parseAsync(inputs);
- const [hostIp] = await verifyHostInputValidity({ host: providerInputs.host, isDynamicSecret: true });
- return { ...providerInputs, hostIp };
+ await Promise.all(
+ providerInputs.bootstrapServers.map(({ host }) => verifyHostInputValidity({ host, isDynamicSecret: true }))
+ );
+ return providerInputs;
};
- // Admin requests are accepted by any KRaft broker, so a single connection to the configured one is enough
+ // KRaft brokers forward admin requests to the controller, so the configured broker is enough. ZooKeeper
+ // clusters only accept SCRAM changes on the controller itself, so there every request goes to it instead.
const $withConnection = async <T>(
providerInputs: TKafkaProviderInputs,
errorPrefix: string,
callback: (connection: Connection) => Promise<T>,
sensitiveTokens: string[] = []
) => {
- const connection = new Connection("infisical", {
- sasl: {
- mechanism: providerInputs.saslMechanism,
- username: providerInputs.username,
- password: providerInputs.password
- },
- ...(providerInputs.sslEnabled && {
- tls: {
- ca: providerInputs.ca || undefined,
- rejectUnauthorized: providerInputs.sslRejectUnauthorized,
- ...getTlsServerNameOptions(providerInputs.host)
- }
- })
- });
+ const connections: Connection[] = [];
+ // the controller's address comes from the broker, so every host gets the same checks before connecting
+ const connect = async (host: string, port: number) => {
+ const [hostIp] = await verifyHostInputValidity({ host, isDynamicSecret: true });
+ const connection = new Connection("infisical", {
+ sasl: {
+ mechanism: providerInputs.saslMechanism,
+ username: providerInputs.username,
+ password: providerInputs.password
+ },
+ ...(providerInputs.sslEnabled && {
+ tls: {
+ ca: providerInputs.ca || undefined,
+ rejectUnauthorized: providerInputs.sslRejectUnauthorized,
+ ...getTlsServerNameOptions(host)
+ }
+ })
+ });
+ connections.push(connection);
+ await connection.connect(hostIp, port);
+ return connection;
+ };
try {
- await connection.connect(providerInputs.hostIp, providerInputs.port);
- return await callback(connection);
+ // like Kafka clients, use the first bootstrap server that connects
+ const connection = await providerInputs.bootstrapServers.reduce<Promise<Connection>>(
+ (previous, { host, port }) => previous.catch(() => connect(host, port)),
+ Promise.reject(new Error("No bootstrap servers configured"))
+ );
+ const { apiKeys, finalizedFeatures } = await apiVersionsV3.api.async(
+ connection,
+ clientSoftwareName,
+ String(clientSoftwareVersion)
+ );
+ const isSupportedCluster = REQUIRED_APIS.every(({ key, version }) =>
+ apiKeys.some(
+ ({ apiKey, minVersion, maxVersion }) => apiKey === key && minVersion <= version && version <= maxVersion
+ )
+ );
+ if (!isSupportedCluster) {
+ throw new BadRequestError({
+ message: "Kafka dynamic secrets require Kafka 2.7 or later with ZooKeeper, or 3.5 or later with KRaft."
+ });
+ }
+ // only KRaft clusters report metadata.version
+ if (finalizedFeatures?.some(({ name }) => name === "metadata.version")) return await callback(connection);
+
+ const { brokers, controllerId } = await metadataV9.api.async(connection, []);
+ const controller = brokers.find(({ nodeId }) => nodeId === controllerId);
+ if (!controller) throw new BadRequestError({ message: "The Kafka cluster has no active controller." });
+ return await callback(await connect(controller.host, controller.port));
} catch (err) {
const sanitizedErrorMessage = sanitizeString({
unsanitizedString: getErrorMessage(err),
- tokens: [...sensitiveTokens, providerInputs.password, providerInputs.username, providerInputs.host]
+ tokens: [
+ ...sensitiveTokens,
+ providerInputs.password,
+ providerInputs.username,
+ ...providerInputs.bootstrapServers.map(({ host }) => host)
+ ]
});
throw new BadRequestError({ message: `${errorPrefix}: ${sanitizedErrorMessage}` });
} finally {
- await connection.close();
+ await Promise.all(connections.map((connection) => connection.close()));
}
};
const validateConnection = async (inputs: unknown) => {
const providerInputs = await validateProviderInputs(inputs);
- const { isSupportedCluster, isAuthorizerEnabled } = await $withConnection(
- providerInputs,
- "Failed to connect with provider",
- async (connection) => {
- const { apiKeys, finalizedFeatures } = await apiVersionsV3.api.async(
- connection,
- clientSoftwareName,
- String(clientSoftwareVersion)
- );
-
- return {
- // only KRaft clusters report metadata.version, and they support managing SCRAM users from Kafka 3.5
- isSupportedCluster:
- Boolean(finalizedFeatures?.some(({ name }) => name === "metadata.version")) &&
- REQUIRED_APIS.every(({ key, version }) =>
- apiKeys.some(
- ({ apiKey, minVersion, maxVersion }) => apiKey === key && minVersion <= version && version <= maxVersion
- )
- ),
- isAuthorizerEnabled: await describeAclsV3.api
- .async(connection, getPrincipalAclFilter(providerInputs.username))
- .then(
- () => true,
- (err: Error) => {
- if (findErrorBy(err, "apiId", "SECURITY_DISABLED")) return false;
- throw err;
- }
- )
- };
- }
+ const isAuthorizerEnabled = await $withConnection(providerInputs, "Failed to connect with provider", (connection) =>
+ describeAclsV3.api.async(connection, getPrincipalAclFilter(providerInputs.username)).then(
+ () => true,
+ (err: Error) => {
+ if (findErrorBy(err, "apiId", "SECURITY_DISABLED")) return false;
+ throw err;
+ }
+ )
);
- if (!isSupportedCluster) {
- throw new BadRequestError({
- message: "Kafka dynamic secrets require Kafka 3.5 or later running in KRaft mode."
- });
- }
if (!isAuthorizerEnabled) {
throw new BadRequestError({
message:
diff --git a/backend/src/ee/services/dynamic-secret/providers/models.ts b/backend/src/ee/services/dynamic-secret/providers/models.ts
index f1298c9ef9..439db55dfe 100644
--- a/backend/src/ee/services/dynamic-secret/providers/models.ts
+++ b/backend/src/ee/services/dynamic-secret/providers/models.ts
@@ -1140,8 +1140,10 @@ export enum KafkaAclPermissionType {
const KAFKA_CLUSTER_RESOURCE_NAME = "kafka-cluster";
export const DynamicSecretKafkaSchema = z.object({
- host: z.string().trim().min(1).max(255).describe("Host of a Kafka broker"),
- port: z.number().int().min(1).max(65535),
+ bootstrapServers: z
+ .array(z.object({ host: z.string().trim().min(1).max(255), port: z.number().int().min(1).max(65535) }))
+ .min(1)
+ .describe("Brokers Infisical tries in order until one connects"),
saslMechanism: z
.nativeEnum(KafkaSaslMechanism)
.describe("SASL mechanism Infisical uses to authenticate as the admin user"),
diff --git a/frontend/src/components/dynamic-secrets/DynamicSecretProviderForm/data-protocol-contract.test.ts b/frontend/src/components/dynamic-secrets/DynamicSecretProviderForm/data-protocol-contract.test.ts
index cd1659ace6..57d4c23321 100644
--- a/frontend/src/components/dynamic-secrets/DynamicSecretProviderForm/data-protocol-contract.test.ts
+++ b/frontend/src/components/dynamic-secrets/DynamicSecretProviderForm/data-protocol-contract.test.ts
@@ -848,7 +848,7 @@ const kafkaCreateValues = {
name: "kafka-secret",
inputs: {
...kafkaCreateDefaults.inputs,
- host: "kafka.example.com",
+ bootstrapServers: [{ host: "kafka.example.com", port: 9092 }],
username: "admin",
password: "secret",
acls: [{ ...getDefaultKafkaAcl(), resourceName: "orders" }]
diff --git a/frontend/src/components/dynamic-secrets/DynamicSecretProviderForm/providerDefinitions/kafka.tsx b/frontend/src/components/dynamic-secrets/DynamicSecretProviderForm/providerDefinitions/kafka.tsx
index f0db92fd29..3e5f6a7664 100644
--- a/frontend/src/components/dynamic-secrets/DynamicSecretProviderForm/providerDefinitions/kafka.tsx
+++ b/frontend/src/components/dynamic-secrets/DynamicSecretProviderForm/providerDefinitions/kafka.tsx
@@ -1,4 +1,4 @@
-import { Controller, useFieldArray, useFormContext } from "react-hook-form";
+import { Controller, FieldArray, Path, useFieldArray, useFormContext } from "react-hook-form";
import { PlusIcon, Trash2Icon } from "lucide-react";
import {
@@ -14,6 +14,7 @@ import {
SelectTrigger,
SelectValue
} from "@app/components/v3";
+import { cn } from "@app/components/v3/utils";
import {
DynamicSecretProviders,
KafkaAclOperation,
@@ -27,13 +28,10 @@ import { DynamicSecretProviderFields } from "../DynamicSecretProviderFields";
import { DynamicSecretProviderGroup } from "../DynamicSecretProviderGroup";
import { DEFAULT_DYNAMIC_SECRET_USERNAME_TEMPLATE } from "../schemas";
import { SslRejectUnauthorizedField } from "../shared";
-import {
- defineDynamicSecretProvider,
- TDynamicSecretProviderField,
- TDynamicSecretProviderFormItem
-} from "../types";
+import { defineDynamicSecretProvider, TDynamicSecretProviderField } from "../types";
import {
getDefaultKafkaAcl,
+ getDefaultKafkaBootstrapServer,
getKafkaCreateDefaultValues,
getKafkaCreatePayload,
getKafkaEditDefaultValues,
@@ -45,15 +43,6 @@ import {
} from "./kafkaContract";
const kafkaConnectionFields = [
- {
- name: "inputs.host",
- type: "text",
- label: "Host",
- placeholder: "kafka.example.com",
- description: "Any broker in the cluster.",
- layout: "half"
- },
- { name: "inputs.port", type: "number", label: "Port", placeholder: "9092", layout: "half" },
{
name: "inputs.saslMechanism",
type: "select",
@@ -103,6 +92,19 @@ const caField = {
rows: 3
} satisfies TDynamicSecretProviderField<TKafkaFormValues>;
+type TKafkaRowColumn = {
+ key: string;
+ label: string;
+ placeholder?: string;
+ type?: "number";
+ options?: readonly { label: string; value: string }[];
+};
+
+const bootstrapServerColumns = [
+ { key: "host", label: "Host", placeholder: "kafka-1.example.com" },
+ { key: "port", label: "Port", placeholder: "9092", type: "number" }
+] satisfies readonly TKafkaRowColumn[];
+
const aclColumns = [
{
key: "resourceType",
@@ -122,7 +124,7 @@ const aclColumns = [
{ label: "Prefixed", value: KafkaAclPatternType.Prefixed }
]
},
- { key: "resourceName", label: "Resource Name" },
+ { key: "resourceName", label: "Resource Name", placeholder: "orders" },
{
key: "operation",
label: "Operation",
@@ -147,24 +149,133 @@ const aclColumns = [
{ label: "Deny", value: KafkaAclPermissionType.Deny }
]
}
-] as const;
+] satisfies readonly TKafkaRowColumn[];
-const kafkaFormFields = [
- {
- kind: "group",
- id: "kafka-connection",
- presentation: "panel",
- fields: kafkaConnectionFields
- }
-] satisfies readonly TDynamicSecretProviderFormItem<TKafkaFormValues>[];
+const KafkaRows = <TName extends "inputs.bootstrapServers" | "inputs.acls">({
+ name,
+ columns,
+ gridClassName,
+ getNewRow,
+ itemLabel
+}: {
+ name: TName;
+ columns: readonly TKafkaRowColumn[];
+ gridClassName: string;
+ getNewRow: () => FieldArray<TKafkaFormValues, TName>;
+ itemLabel: string;
+}) => {
+ const { control } = useFormContext<TKafkaFormValues>();
+ const rows = useFieldArray<TKafkaFormValues, TName>({ control, name });
+
+ return (
+ <div className="flex flex-col gap-3">
+ {rows.fields.map(({ id }, index) => (
+ <div key={id} className={cn("grid grid-cols-1 items-start gap-3", gridClassName)}>
+ {columns.map((column) => {
+ const fieldName = `${name}.${index}.${column.key}`;
+ const inputId = fieldName.replaceAll(".", "-");
+ return (
+ <Controller
+ key={column.key}
+ control={control}
+ name={fieldName as Path<TKafkaFormValues>}
+ render={({ field, fieldState: { error } }) => (
+ <Field data-invalid={Boolean(error)}>
+ <FieldLabel htmlFor={inputId}>{column.label}</FieldLabel>
+ {column.options ? (
+ <Select
+ value={String(field.value)}
+ onValueChange={(value) => {
+ if (!value || value === field.value) return;
+ field.onChange(value);
+ }}
+ >
+ <SelectTrigger
+ ref={field.ref}
+ id={inputId}
+ onBlur={field.onBlur}
+ isError={Boolean(error)}
+ >
+ <SelectValue />
+ </SelectTrigger>
+ <SelectContent>
+ {column.options.map((option) => (
+ <SelectItem key={option.value} value={option.value}>
+ {option.label}
+ </SelectItem>
+ ))}
+ </SelectContent>
+ </Select>
+ ) : (
+ <Input
+ {...field}
+ value={field.value as string | number}
+ id={inputId}
+ type={column.type}
+ placeholder={column.placeholder}
+ isError={Boolean(error)}
+ />
+ )}
+ <FieldError>{error?.message}</FieldError>
+ </Field>
+ )}
+ />
+ );
+ })}
+ <div className="flex flex-col gap-2">
+ <FieldLabel className="pointer-events-none invisible select-none" aria-hidden>
+ &nbsp;
+ </FieldLabel>
+ <IconButton
+ type="button"
+ variant="outline"
+ aria-label={`Remove ${itemLabel} ${index + 1}`}
+ disabled={rows.fields.length === 1}
+ onClick={() => rows.remove(index)}
+ >
+ <Trash2Icon />
+ </IconButton>
+ </div>
+ </div>
+ ))}
+ <Button
+ type="button"
+ size="sm"
+ className="self-start"
+ onClick={() => rows.append(getNewRow())}
+ >
+ <PlusIcon /> Add {itemLabel}
+ </Button>
+ </div>
+ );
+};
const KafkaFields = () => {
- const { control, watch } = useFormContext<TKafkaFormValues>();
- const acls = useFieldArray({ control, name: "inputs.acls" });
+ const { watch } = useFormContext<TKafkaFormValues>();
const sslEnabled = watch("inputs.sslEnabled");
return (
<>
+ <DynamicSecretProviderGroup
+ id="kafka-bootstrap-servers"
+ presentation="panel"
+ surface
+ title="Bootstrap Servers"
+ description="Brokers Infisical tries in order until one connects."
+ >
+ <KafkaRows
+ name="inputs.bootstrapServers"
+ columns={bootstrapServerColumns}
+ gridClassName="sm:grid-cols-[minmax(0,1fr)_8rem_auto]"
+ getNewRow={getDefaultKafkaBootstrapServer}
+ itemLabel="Server"
+ />
+ </DynamicSecretProviderGroup>
+
+ <DynamicSecretProviderGroup id="kafka-connection" presentation="panel">
+ <DynamicSecretProviderFields fields={kafkaConnectionFields} />
+ </DynamicSecretProviderGroup>
+
<DynamicSecretProviderGroup
id="kafka-acls"
presentation="panel"
@@ -172,84 +283,13 @@ const KafkaFields = () => {
title="ACLs"
description="Granted to each lease user. Kafka denies anything these ACLs do not allow."
>
- <div className="flex flex-col gap-3">
- {acls.fields.map(({ id }, index) => (
- <div
- key={id}
- className="grid grid-cols-1 items-start gap-3 sm:grid-cols-[8rem_7rem_minmax(0,1fr)_9rem_6rem_auto]"
- >
- {aclColumns.map((column) => (
- <Controller
- key={column.key}
- control={control}
- name={`inputs.acls.${index}.${column.key}`}
- render={({ field, fieldState: { error } }) => (
- <Field data-invalid={Boolean(error)}>
- <FieldLabel htmlFor={`kafka-acl-${index}-${column.key}`}>
- {column.label}
- </FieldLabel>
- {"options" in column ? (
- <Select
- value={field.value}
- onValueChange={(value) => {
- if (!value || value === field.value) return;
- field.onChange(value);
- }}
- >
- <SelectTrigger
- ref={field.ref}
- id={`kafka-acl-${index}-${column.key}`}
- onBlur={field.onBlur}
- isError={Boolean(error)}
- >
- <SelectValue />
- </SelectTrigger>
- <SelectContent>
- {column.options.map((option) => (
- <SelectItem key={option.value} value={option.value}>
- {option.label}
- </SelectItem>
- ))}
- </SelectContent>
- </Select>
- ) : (
- <Input
- {...field}
- id={`kafka-acl-${index}-${column.key}`}
- placeholder="orders"
- isError={Boolean(error)}
- />
- )}
- <FieldError>{error?.message}</FieldError>
- </Field>
- )}
- />
- ))}
- <div className="flex flex-col gap-2">
- <FieldLabel className="pointer-events-none invisible select-none" aria-hidden>
- &nbsp;
- </FieldLabel>
- <IconButton
- type="button"
- variant="outline"
- aria-label={`Remove ACL ${index + 1}`}
- disabled={acls.fields.length === 1}
- onClick={() => acls.remove(index)}
- >
- <Trash2Icon />
- </IconButton>
- </div>
- </div>
- ))}
- <Button
- type="button"
- size="sm"
- className="self-start"
- onClick={() => acls.append(getDefaultKafkaAcl())}
- >
- <PlusIcon /> Add ACL
- </Button>
- </div>
+ <KafkaRows
+ name="inputs.acls"
+ columns={aclColumns}
+ gridClassName="sm:grid-cols-[8rem_7rem_minmax(0,1fr)_9rem_6rem_auto]"
+ getNewRow={getDefaultKafkaAcl}
+ itemLabel="ACL"
+ />
</DynamicSecretProviderGroup>
<DynamicSecretProviderGroup id="kafka-advanced" presentation="collapse" title="Advanced">
@@ -265,7 +305,6 @@ const KafkaFields = () => {
export const kafkaDynamicSecretProvider = defineDynamicSecretProvider({
provider: DynamicSecretProviders.Kafka,
label: "Kafka",
- fields: kafkaFormFields,
customRenderer: {
reasons: KAFKA_CUSTOM_RENDERER_REASONS,
Component: KafkaFields
diff --git a/frontend/src/components/dynamic-secrets/DynamicSecretProviderForm/providerDefinitions/kafkaContract.ts b/frontend/src/components/dynamic-secrets/DynamicSecretProviderForm/providerDefinitions/kafkaContract.ts
index f326e11498..a3d5aa9387 100644
--- a/frontend/src/components/dynamic-secrets/DynamicSecretProviderForm/providerDefinitions/kafkaContract.ts
+++ b/frontend/src/components/dynamic-secrets/DynamicSecretProviderForm/providerDefinitions/kafkaContract.ts
@@ -34,9 +34,15 @@ const aclSchema = z.object({
permissionType: z.nativeEnum(KafkaAclPermissionType)
});
+const bootstrapServerSchema = z.object({
+ host: z.string().trim().min(1, "Host is required"),
+ port: z.coerce.number()
+});
+
export const kafkaCreateInputsSchema = z.object({
- host: z.string().trim().min(1),
- port: z.coerce.number(),
+ bootstrapServers: z
+ .array(bootstrapServerSchema)
+ .min(1, "At least one bootstrap server is required"),
saslMechanism: z.nativeEnum(KafkaSaslMechanism),
username: z.string().trim().min(1),
password: z.string().trim().min(1),
@@ -61,6 +67,11 @@ export const kafkaEditFormSchema = editDynamicSecretProviderFormSchema(kafkaEdit
usernameTemplateSchema: z.string().trim().nullable().optional()
}) as z.ZodType<TKafkaFormValues>;
+export const getDefaultKafkaBootstrapServer = (): TKafkaFormInputs["bootstrapServers"][number] => ({
+ host: "",
+ port: 9092
+});
+
export const getDefaultKafkaAcl = (): TKafkaFormInputs["acls"][number] => ({
resourceType: KafkaAclResourceType.Topic,
patternType: KafkaAclPatternType.Literal,
@@ -78,8 +89,7 @@ export const getKafkaCreateDefaultValues = (
environment: context.isSingleEnvironmentMode ? context.environments[0] : undefined,
usernameTemplate: DEFAULT_DYNAMIC_SECRET_USERNAME_TEMPLATE,
inputs: {
- host: "",
- port: 9092,
+ bootstrapServers: [getDefaultKafkaBootstrapServer()],
saslMechanism: KafkaSaslMechanism.ScramSha512,
username: "",
password: "",
diff --git a/frontend/src/hooks/api/dynamicSecret/types.ts b/frontend/src/hooks/api/dynamicSecret/types.ts
index 86ae97323d..9522298058 100644
--- a/frontend/src/hooks/api/dynamicSecret/types.ts
+++ b/frontend/src/hooks/api/dynamicSecret/types.ts
@@ -642,8 +642,7 @@ export type TDynamicSecretProvider =
| {
type: DynamicSecretProviders.Kafka;
inputs: {
- host: string;
- port: number;
+ bootstrapServers: { host: string; port: number }[];
saslMechanism: KafkaSaslMechanism;
username: string;
password: string;