Infisical is the open-source platform for securing secrets, certificates, access, and AI Agents.
TypeScript acmeagentsai-agentscertificate-managementdevsecopsenvironment-variableskubernetesopen-sourcepampkipostgresprivate-caprivileged-access-managementsecret-managementsecret-managersecretssecrets-managementsecuritysecurity-toolsvault
feat(dynamic-secrets): support ZooKeeper Kafka clusters and multiple bootstrap servers
Dimitris Staratzis2bbe36660617
diff --git a/backend/src/ee/services/dynamic-secret/providers/kafka.ts b/backend/src/ee/services/dynamic-secret/providers/kafka.tsindex 4dbe29838a..2a4f3376a5 100644--- a/backend/src/ee/services/dynamic-secret/providers/kafka.ts+++ b/backend/src/ee/services/dynamic-secret/providers/kafka.ts@@ -14,6 +14,7 @@ import {deleteAclsV3,describeAclsV3,findErrorBy,+ metadataV9,ResourcePatternTypes,ResourceTypes,ScramMechanisms@@ -32,7 +33,7 @@ import { verifyHostInputValidity } from "../dynamic-secret-fns";import { DynamicSecretKafkaSchema, TDynamicProviderFns } from "./models";import { generateUsername } from "./templateUtils";-type TKafkaProviderInputs = z.infer<typeof DynamicSecretKafkaSchema> & { hostIp: string };+type TKafkaProviderInputs = z.infer<typeof DynamicSecretKafkaSchema>;const pbkdf2Async = promisify(pbkdf2);@@ -91,86 +92,100 @@ const deleteKafkaUser = async (connection: Connection, username: string) => {export const KafkaProvider = (): TDynamicProviderFns => {const validateProviderInputs = async (inputs: unknown) => {const providerInputs = await DynamicSecretKafkaSchema.parseAsync(inputs);- const [hostIp] = await verifyHostInputValidity({ host: providerInputs.host, isDynamicSecret: true });- return { ...providerInputs, hostIp };+ await Promise.all(+ providerInputs.bootstrapServers.map(({ host }) => verifyHostInputValidity({ host, isDynamicSecret: true }))+ );+ return providerInputs;};- // Admin requests are accepted by any KRaft broker, so a single connection to the configured one is enough+ // KRaft brokers forward admin requests to the controller, so the configured broker is enough. ZooKeeper+ // clusters only accept SCRAM changes on the controller itself, so there every request goes to it instead.const $withConnection = async <T>(providerInputs: TKafkaProviderInputs,errorPrefix: string,callback: (connection: Connection) => Promise<T>,sensitiveTokens: string[] = []) => {- const connection = new Connection("infisical", {- sasl: {- mechanism: providerInputs.saslMechanism,- username: providerInputs.username,- password: providerInputs.password- },- ...(providerInputs.sslEnabled && {- tls: {- ca: providerInputs.ca || undefined,- rejectUnauthorized: providerInputs.sslRejectUnauthorized,- ...getTlsServerNameOptions(providerInputs.host)- }- })- });+ const connections: Connection[] = [];+ // the controller's address comes from the broker, so every host gets the same checks before connecting+ const connect = async (host: string, port: number) => {+ const [hostIp] = await verifyHostInputValidity({ host, isDynamicSecret: true });+ const connection = new Connection("infisical", {+ sasl: {+ mechanism: providerInputs.saslMechanism,+ username: providerInputs.username,+ password: providerInputs.password+ },+ ...(providerInputs.sslEnabled && {+ tls: {+ ca: providerInputs.ca || undefined,+ rejectUnauthorized: providerInputs.sslRejectUnauthorized,+ ...getTlsServerNameOptions(host)+ }+ })+ });+ connections.push(connection);+ await connection.connect(hostIp, port);+ return connection;+ };try {- await connection.connect(providerInputs.hostIp, providerInputs.port);- return await callback(connection);+ // like Kafka clients, use the first bootstrap server that connects+ const connection = await providerInputs.bootstrapServers.reduce<Promise<Connection>>(+ (previous, { host, port }) => previous.catch(() => connect(host, port)),+ Promise.reject(new Error("No bootstrap servers configured"))+ );+ const { apiKeys, finalizedFeatures } = await apiVersionsV3.api.async(+ connection,+ clientSoftwareName,+ String(clientSoftwareVersion)+ );+ const isSupportedCluster = REQUIRED_APIS.every(({ key, version }) =>+ apiKeys.some(+ ({ apiKey, minVersion, maxVersion }) => apiKey === key && minVersion <= version && version <= maxVersion+ )+ );+ if (!isSupportedCluster) {+ throw new BadRequestError({+ message: "Kafka dynamic secrets require Kafka 2.7 or later with ZooKeeper, or 3.5 or later with KRaft."+ });+ }+ // only KRaft clusters report metadata.version+ if (finalizedFeatures?.some(({ name }) => name === "metadata.version")) return await callback(connection);++ const { brokers, controllerId } = await metadataV9.api.async(connection, []);+ const controller = brokers.find(({ nodeId }) => nodeId === controllerId);+ if (!controller) throw new BadRequestError({ message: "The Kafka cluster has no active controller." });+ return await callback(await connect(controller.host, controller.port));} catch (err) {const sanitizedErrorMessage = sanitizeString({unsanitizedString: getErrorMessage(err),- tokens: [...sensitiveTokens, providerInputs.password, providerInputs.username, providerInputs.host]+ tokens: [+ ...sensitiveTokens,+ providerInputs.password,+ providerInputs.username,+ ...providerInputs.bootstrapServers.map(({ host }) => host)+ ]});throw new BadRequestError({ message: `${errorPrefix}: ${sanitizedErrorMessage}` });} finally {- await connection.close();+ await Promise.all(connections.map((connection) => connection.close()));}};const validateConnection = async (inputs: unknown) => {const providerInputs = await validateProviderInputs(inputs);- const { isSupportedCluster, isAuthorizerEnabled } = await $withConnection(- providerInputs,- "Failed to connect with provider",- async (connection) => {- const { apiKeys, finalizedFeatures } = await apiVersionsV3.api.async(- connection,- clientSoftwareName,- String(clientSoftwareVersion)- );-- return {- // only KRaft clusters report metadata.version, and they support managing SCRAM users from Kafka 3.5- isSupportedCluster:- Boolean(finalizedFeatures?.some(({ name }) => name === "metadata.version")) &&- REQUIRED_APIS.every(({ key, version }) =>- apiKeys.some(- ({ apiKey, minVersion, maxVersion }) => apiKey === key && minVersion <= version && version <= maxVersion- )- ),- isAuthorizerEnabled: await describeAclsV3.api- .async(connection, getPrincipalAclFilter(providerInputs.username))- .then(- () => true,- (err: Error) => {- if (findErrorBy(err, "apiId", "SECURITY_DISABLED")) return false;- throw err;- }- )- };- }+ const isAuthorizerEnabled = await $withConnection(providerInputs, "Failed to connect with provider", (connection) =>+ describeAclsV3.api.async(connection, getPrincipalAclFilter(providerInputs.username)).then(+ () => true,+ (err: Error) => {+ if (findErrorBy(err, "apiId", "SECURITY_DISABLED")) return false;+ throw err;+ }+ ));- if (!isSupportedCluster) {- throw new BadRequestError({- message: "Kafka dynamic secrets require Kafka 3.5 or later running in KRaft mode."- });- }if (!isAuthorizerEnabled) {throw new BadRequestError({message:diff --git a/backend/src/ee/services/dynamic-secret/providers/models.ts b/backend/src/ee/services/dynamic-secret/providers/models.tsindex f1298c9ef9..439db55dfe 100644--- a/backend/src/ee/services/dynamic-secret/providers/models.ts+++ b/backend/src/ee/services/dynamic-secret/providers/models.ts@@ -1140,8 +1140,10 @@ export enum KafkaAclPermissionType {const KAFKA_CLUSTER_RESOURCE_NAME = "kafka-cluster";export const DynamicSecretKafkaSchema = z.object({- host: z.string().trim().min(1).max(255).describe("Host of a Kafka broker"),- port: z.number().int().min(1).max(65535),+ bootstrapServers: z+ .array(z.object({ host: z.string().trim().min(1).max(255), port: z.number().int().min(1).max(65535) }))+ .min(1)+ .describe("Brokers Infisical tries in order until one connects"),saslMechanism: z.nativeEnum(KafkaSaslMechanism).describe("SASL mechanism Infisical uses to authenticate as the admin user"),diff --git a/frontend/src/components/dynamic-secrets/DynamicSecretProviderForm/data-protocol-contract.test.ts b/frontend/src/components/dynamic-secrets/DynamicSecretProviderForm/data-protocol-contract.test.tsindex cd1659ace6..57d4c23321 100644--- a/frontend/src/components/dynamic-secrets/DynamicSecretProviderForm/data-protocol-contract.test.ts+++ b/frontend/src/components/dynamic-secrets/DynamicSecretProviderForm/data-protocol-contract.test.ts@@ -848,7 +848,7 @@ const kafkaCreateValues = {name: "kafka-secret",inputs: {...kafkaCreateDefaults.inputs,- host: "kafka.example.com",+ bootstrapServers: [{ host: "kafka.example.com", port: 9092 }],username: "admin",password: "secret",acls: [{ ...getDefaultKafkaAcl(), resourceName: "orders" }]diff --git a/frontend/src/components/dynamic-secrets/DynamicSecretProviderForm/providerDefinitions/kafka.tsx b/frontend/src/components/dynamic-secrets/DynamicSecretProviderForm/providerDefinitions/kafka.tsxindex f0db92fd29..3e5f6a7664 100644--- a/frontend/src/components/dynamic-secrets/DynamicSecretProviderForm/providerDefinitions/kafka.tsx+++ b/frontend/src/components/dynamic-secrets/DynamicSecretProviderForm/providerDefinitions/kafka.tsx@@ -1,4 +1,4 @@-import { Controller, useFieldArray, useFormContext } from "react-hook-form";+import { Controller, FieldArray, Path, useFieldArray, useFormContext } from "react-hook-form";import { PlusIcon, Trash2Icon } from "lucide-react";import {@@ -14,6 +14,7 @@ import {SelectTrigger,SelectValue} from "@app/components/v3";+import { cn } from "@app/components/v3/utils";import {DynamicSecretProviders,KafkaAclOperation,@@ -27,13 +28,10 @@ import { DynamicSecretProviderFields } from "../DynamicSecretProviderFields";import { DynamicSecretProviderGroup } from "../DynamicSecretProviderGroup";import { DEFAULT_DYNAMIC_SECRET_USERNAME_TEMPLATE } from "../schemas";import { SslRejectUnauthorizedField } from "../shared";-import {- defineDynamicSecretProvider,- TDynamicSecretProviderField,- TDynamicSecretProviderFormItem-} from "../types";+import { defineDynamicSecretProvider, TDynamicSecretProviderField } from "../types";import {getDefaultKafkaAcl,+ getDefaultKafkaBootstrapServer,getKafkaCreateDefaultValues,getKafkaCreatePayload,getKafkaEditDefaultValues,@@ -45,15 +43,6 @@ import {} from "./kafkaContract";const kafkaConnectionFields = [- {- name: "inputs.host",- type: "text",- label: "Host",- placeholder: "kafka.example.com",- description: "Any broker in the cluster.",- layout: "half"- },- { name: "inputs.port", type: "number", label: "Port", placeholder: "9092", layout: "half" },{name: "inputs.saslMechanism",type: "select",@@ -103,6 +92,19 @@ const caField = {rows: 3} satisfies TDynamicSecretProviderField<TKafkaFormValues>;+type TKafkaRowColumn = {+ key: string;+ label: string;+ placeholder?: string;+ type?: "number";+ options?: readonly { label: string; value: string }[];+};++const bootstrapServerColumns = [+ { key: "host", label: "Host", placeholder: "kafka-1.example.com" },+ { key: "port", label: "Port", placeholder: "9092", type: "number" }+] satisfies readonly TKafkaRowColumn[];+const aclColumns = [{key: "resourceType",@@ -122,7 +124,7 @@ const aclColumns = [{ label: "Prefixed", value: KafkaAclPatternType.Prefixed }]},- { key: "resourceName", label: "Resource Name" },+ { key: "resourceName", label: "Resource Name", placeholder: "orders" },{key: "operation",label: "Operation",@@ -147,24 +149,133 @@ const aclColumns = [{ label: "Deny", value: KafkaAclPermissionType.Deny }]}-] as const;+] satisfies readonly TKafkaRowColumn[];-const kafkaFormFields = [- {- kind: "group",- id: "kafka-connection",- presentation: "panel",- fields: kafkaConnectionFields- }-] satisfies readonly TDynamicSecretProviderFormItem<TKafkaFormValues>[];+const KafkaRows = <TName extends "inputs.bootstrapServers" | "inputs.acls">({+ name,+ columns,+ gridClassName,+ getNewRow,+ itemLabel+}: {+ name: TName;+ columns: readonly TKafkaRowColumn[];+ gridClassName: string;+ getNewRow: () => FieldArray<TKafkaFormValues, TName>;+ itemLabel: string;+}) => {+ const { control } = useFormContext<TKafkaFormValues>();+ const rows = useFieldArray<TKafkaFormValues, TName>({ control, name });++ return (+ <div className="flex flex-col gap-3">+ {rows.fields.map(({ id }, index) => (+ <div key={id} className={cn("grid grid-cols-1 items-start gap-3", gridClassName)}>+ {columns.map((column) => {+ const fieldName = `${name}.${index}.${column.key}`;+ const inputId = fieldName.replaceAll(".", "-");+ return (+ <Controller+ key={column.key}+ control={control}+ name={fieldName as Path<TKafkaFormValues>}+ render={({ field, fieldState: { error } }) => (+ <Field data-invalid={Boolean(error)}>+ <FieldLabel htmlFor={inputId}>{column.label}</FieldLabel>+ {column.options ? (+ <Select+ value={String(field.value)}+ onValueChange={(value) => {+ if (!value || value === field.value) return;+ field.onChange(value);+ }}+ >+ <SelectTrigger+ ref={field.ref}+ id={inputId}+ onBlur={field.onBlur}+ isError={Boolean(error)}+ >+ <SelectValue />+ </SelectTrigger>+ <SelectContent>+ {column.options.map((option) => (+ <SelectItem key={option.value} value={option.value}>+ {option.label}+ </SelectItem>+ ))}+ </SelectContent>+ </Select>+ ) : (+ <Input+ {...field}+ value={field.value as string | number}+ id={inputId}+ type={column.type}+ placeholder={column.placeholder}+ isError={Boolean(error)}+ />+ )}+ <FieldError>{error?.message}</FieldError>+ </Field>+ )}+ />+ );+ })}+ <div className="flex flex-col gap-2">+ <FieldLabel className="pointer-events-none invisible select-none" aria-hidden>+ + </FieldLabel>+ <IconButton+ type="button"+ variant="outline"+ aria-label={`Remove ${itemLabel} ${index + 1}`}+ disabled={rows.fields.length === 1}+ onClick={() => rows.remove(index)}+ >+ <Trash2Icon />+ </IconButton>+ </div>+ </div>+ ))}+ <Button+ type="button"+ size="sm"+ className="self-start"+ onClick={() => rows.append(getNewRow())}+ >+ <PlusIcon /> Add {itemLabel}+ </Button>+ </div>+ );+};const KafkaFields = () => {- const { control, watch } = useFormContext<TKafkaFormValues>();- const acls = useFieldArray({ control, name: "inputs.acls" });+ const { watch } = useFormContext<TKafkaFormValues>();const sslEnabled = watch("inputs.sslEnabled");return (<>+ <DynamicSecretProviderGroup+ id="kafka-bootstrap-servers"+ presentation="panel"+ surface+ title="Bootstrap Servers"+ description="Brokers Infisical tries in order until one connects."+ >+ <KafkaRows+ name="inputs.bootstrapServers"+ columns={bootstrapServerColumns}+ gridClassName="sm:grid-cols-[minmax(0,1fr)_8rem_auto]"+ getNewRow={getDefaultKafkaBootstrapServer}+ itemLabel="Server"+ />+ </DynamicSecretProviderGroup>++ <DynamicSecretProviderGroup id="kafka-connection" presentation="panel">+ <DynamicSecretProviderFields fields={kafkaConnectionFields} />+ </DynamicSecretProviderGroup>+<DynamicSecretProviderGroupid="kafka-acls"presentation="panel"@@ -172,84 +283,13 @@ const KafkaFields = () => {title="ACLs"description="Granted to each lease user. Kafka denies anything these ACLs do not allow.">- <div className="flex flex-col gap-3">- {acls.fields.map(({ id }, index) => (- <div- key={id}- className="grid grid-cols-1 items-start gap-3 sm:grid-cols-[8rem_7rem_minmax(0,1fr)_9rem_6rem_auto]"- >- {aclColumns.map((column) => (- <Controller- key={column.key}- control={control}- name={`inputs.acls.${index}.${column.key}`}- render={({ field, fieldState: { error } }) => (- <Field data-invalid={Boolean(error)}>- <FieldLabel htmlFor={`kafka-acl-${index}-${column.key}`}>- {column.label}- </FieldLabel>- {"options" in column ? (- <Select- value={field.value}- onValueChange={(value) => {- if (!value || value === field.value) return;- field.onChange(value);- }}- >- <SelectTrigger- ref={field.ref}- id={`kafka-acl-${index}-${column.key}`}- onBlur={field.onBlur}- isError={Boolean(error)}- >- <SelectValue />- </SelectTrigger>- <SelectContent>- {column.options.map((option) => (- <SelectItem key={option.value} value={option.value}>- {option.label}- </SelectItem>- ))}- </SelectContent>- </Select>- ) : (- <Input- {...field}- id={`kafka-acl-${index}-${column.key}`}- placeholder="orders"- isError={Boolean(error)}- />- )}- <FieldError>{error?.message}</FieldError>- </Field>- )}- />- ))}- <div className="flex flex-col gap-2">- <FieldLabel className="pointer-events-none invisible select-none" aria-hidden>- - </FieldLabel>- <IconButton- type="button"- variant="outline"- aria-label={`Remove ACL ${index + 1}`}- disabled={acls.fields.length === 1}- onClick={() => acls.remove(index)}- >- <Trash2Icon />- </IconButton>- </div>- </div>- ))}- <Button- type="button"- size="sm"- className="self-start"- onClick={() => acls.append(getDefaultKafkaAcl())}- >- <PlusIcon /> Add ACL- </Button>- </div>+ <KafkaRows+ name="inputs.acls"+ columns={aclColumns}+ gridClassName="sm:grid-cols-[8rem_7rem_minmax(0,1fr)_9rem_6rem_auto]"+ getNewRow={getDefaultKafkaAcl}+ itemLabel="ACL"+ /></DynamicSecretProviderGroup><DynamicSecretProviderGroup id="kafka-advanced" presentation="collapse" title="Advanced">@@ -265,7 +305,6 @@ const KafkaFields = () => {export const kafkaDynamicSecretProvider = defineDynamicSecretProvider({provider: DynamicSecretProviders.Kafka,label: "Kafka",- fields: kafkaFormFields,customRenderer: {reasons: KAFKA_CUSTOM_RENDERER_REASONS,Component: KafkaFieldsdiff --git a/frontend/src/components/dynamic-secrets/DynamicSecretProviderForm/providerDefinitions/kafkaContract.ts b/frontend/src/components/dynamic-secrets/DynamicSecretProviderForm/providerDefinitions/kafkaContract.tsindex f326e11498..a3d5aa9387 100644--- a/frontend/src/components/dynamic-secrets/DynamicSecretProviderForm/providerDefinitions/kafkaContract.ts+++ b/frontend/src/components/dynamic-secrets/DynamicSecretProviderForm/providerDefinitions/kafkaContract.ts@@ -34,9 +34,15 @@ const aclSchema = z.object({permissionType: z.nativeEnum(KafkaAclPermissionType)});+const bootstrapServerSchema = z.object({+ host: z.string().trim().min(1, "Host is required"),+ port: z.coerce.number()+});+export const kafkaCreateInputsSchema = z.object({- host: z.string().trim().min(1),- port: z.coerce.number(),+ bootstrapServers: z+ .array(bootstrapServerSchema)+ .min(1, "At least one bootstrap server is required"),saslMechanism: z.nativeEnum(KafkaSaslMechanism),username: z.string().trim().min(1),password: z.string().trim().min(1),@@ -61,6 +67,11 @@ export const kafkaEditFormSchema = editDynamicSecretProviderFormSchema(kafkaEditusernameTemplateSchema: z.string().trim().nullable().optional()}) as z.ZodType<TKafkaFormValues>;+export const getDefaultKafkaBootstrapServer = (): TKafkaFormInputs["bootstrapServers"][number] => ({+ host: "",+ port: 9092+});+export const getDefaultKafkaAcl = (): TKafkaFormInputs["acls"][number] => ({resourceType: KafkaAclResourceType.Topic,patternType: KafkaAclPatternType.Literal,@@ -78,8 +89,7 @@ export const getKafkaCreateDefaultValues = (environment: context.isSingleEnvironmentMode ? context.environments[0] : undefined,usernameTemplate: DEFAULT_DYNAMIC_SECRET_USERNAME_TEMPLATE,inputs: {- host: "",- port: 9092,+ bootstrapServers: [getDefaultKafkaBootstrapServer()],saslMechanism: KafkaSaslMechanism.ScramSha512,username: "",password: "",diff --git a/frontend/src/hooks/api/dynamicSecret/types.ts b/frontend/src/hooks/api/dynamicSecret/types.tsindex 86ae97323d..9522298058 100644--- a/frontend/src/hooks/api/dynamicSecret/types.ts+++ b/frontend/src/hooks/api/dynamicSecret/types.ts@@ -642,8 +642,7 @@ export type TDynamicSecretProvider =| {type: DynamicSecretProviders.Kafka;inputs: {- host: string;- port: number;+ bootstrapServers: { host: string; port: number }[];saslMechanism: KafkaSaslMechanism;username: string;password: string;