Infisical is the open-source platform for securing secrets, certificates, access, and AI Agents.
TypeScript acmeagentsai-agentscertificate-managementdevsecopsenvironment-variableskubernetesopen-sourcepampkipostgresprivate-caprivileged-access-managementsecret-managementsecret-managersecretssecrets-managementsecuritysecurity-toolsvault
fix(acme): lock Cloud DNS records per name, keep existing TTL, show GCP project IDs
bernie-g8ad13b53478b
diff --git a/backend/src/services/certificate-authority/acme/acme-certificate-authority-fns.ts b/backend/src/services/certificate-authority/acme/acme-certificate-authority-fns.tsindex 0ed1ae619d..11424e18b4 100644--- a/backend/src/services/certificate-authority/acme/acme-certificate-authority-fns.ts+++ b/backend/src/services/certificate-authority/acme/acme-certificate-authority-fns.ts@@ -522,7 +522,8 @@ export const executeAcmeOrder = async (connection as TGcpConnection,acmeCa.configuration.dnsProviderConfig.hostedZoneId,recordName,- recordValue+ recordValue,+ keyStore);break;}@@ -601,7 +602,8 @@ export const executeAcmeOrder = async (connection as TGcpConnection,acmeCa.configuration.dnsProviderConfig.hostedZoneId,recordName,- recordValue+ recordValue,+ keyStore);break;}diff --git a/backend/src/services/certificate-authority/acme/dns-providers/dns-record-lock.ts b/backend/src/services/certificate-authority/acme/dns-providers/dns-record-lock.tsnew file mode 100644index 0000000000..97e00a393f--- /dev/null+++ b/backend/src/services/certificate-authority/acme/dns-providers/dns-record-lock.ts@@ -0,0 +1,57 @@+import { KeyStorePrefixes, TKeyStoreFactory } from "@app/keystore/keystore";+import { logger } from "@app/lib/logger";++const RECORD_LOCK_RETRY = { retryCount: 50, retryDelay: 2_000, retryJitter: 300 };++export type TDnsRecordLockKeyStore = Pick<TKeyStoreFactory, "acquireLock">;++const pendingRecordOperations = new Map<string, Promise<void>>();++const withLocalRecordLock = async <T>(key: string, operation: () => Promise<T>): Promise<T> => {+ const pending = pendingRecordOperations.get(key) ?? Promise.resolve();+ const result = pending.then(operation, operation);+ const settled = result.then(+ () => undefined,+ () => undefined+ );++ pendingRecordOperations.set(key, settled);+ void settled.then(() => {+ if (pendingRecordOperations.get(key) === settled) pendingRecordOperations.delete(key);+ });++ return result;+};++export const withDnsRecordLock = async <T>(+ {+ connectionId,+ zoneId,+ name,+ providerName,+ lockTtlMs+ }: { connectionId: string; zoneId: string; name: string; providerName: string; lockTtlMs: number },+ keyStore: TDnsRecordLockKeyStore | undefined,+ operation: () => Promise<T>+): Promise<T> =>+ withLocalRecordLock(`${connectionId}|${zoneId}|${name}`, async () => {+ if (!keyStore) return operation();++ const lock = await keyStore+ .acquireLock([KeyStorePrefixes.AcmeDnsRecordLock(connectionId, zoneId, name)], lockTtlMs, RECORD_LOCK_RETRY)+ .catch(() => null);++ if (!lock) {+ throw new Error(+ `Timed out waiting to update the ${providerName} record '${name}'. Another certificate order is still using it.`+ );+ }++ try {+ return await operation();+ } finally {+ await lock.release().catch((error) => {+ logger.warn(error, `Failed to release ${providerName} record lock [zoneId=${zoneId}] [name=${name}]`);+ });+ }+ });diff --git a/backend/src/services/certificate-authority/acme/dns-providers/gcp-cloud-dns.test.ts b/backend/src/services/certificate-authority/acme/dns-providers/gcp-cloud-dns.test.tsindex 536d1c2bd8..31c7423069 100644--- a/backend/src/services/certificate-authority/acme/dns-providers/gcp-cloud-dns.test.ts+++ b/backend/src/services/certificate-authority/acme/dns-providers/gcp-cloud-dns.test.ts@@ -19,10 +19,12 @@ vi.mock("@app/lib/logger", () => ({logger: { error: vi.fn(), info: vi.fn(), warn: vi.fn(), debug: vi.fn() }}));+// eslint-disable-next-line import/first+import { TDnsRecordLockKeyStore } from "./dns-record-lock";// eslint-disable-next-line import/firstimport { gcpCloudDnsDeleteTxtRecord, gcpCloudDnsInsertTxtRecord, validateGcpCloudDnsZone } from "./gcp-cloud-dns";-const connection = {} as TGcpConnection;+const connection = { id: "connection-id" } as TGcpConnection;const ZONE = "projects/my-project/managedZones/example-zone";const ZONE_URL = `https://dns.googleapis.com/dns/v1/${ZONE}`;const RECORD = "_acme-challenge.example.com";@@ -37,7 +39,7 @@ const axiosError = (status: number, message = `status ${status}`, extra: Record<data: { error: { message, ...extra } }});-const existingRecordSet = (rrdatas: string[]) => ({ data: { name: FQDN, type: "TXT", ttl: 60, rrdatas } });+const existingRecordSet = (rrdatas: string[], ttl = 60) => ({ data: { name: FQDN, type: "TXT", ttl, rrdatas } });describe("validateGcpCloudDnsZone", () => {it("accepts a managed zone resource name", () => {@@ -86,6 +88,82 @@ describe("gcpCloudDnsInsertTxtRecord", () => {);});+ it("keeps the TTL of a record set it did not create", async () => {+ getMock.mockResolvedValueOnce(existingRecordSet(['"customer-value"'], 3600));++ await gcpCloudDnsInsertTxtRecord(connection, ZONE, RECORD, '"token-a"');++ expect(postMock).toHaveBeenCalledWith(+ `${ZONE_URL}/changes`,+ {+ deletions: [{ name: FQDN, type: "TXT", ttl: 3600, rrdatas: ['"customer-value"'] }],+ additions: [{ name: FQDN, type: "TXT", ttl: 3600, rrdatas: ['"customer-value"', '"token-a"'] }]+ },+ expect.anything()+ );+ });++ it("holds a per-record lock while it updates the record set", async () => {+ const release = vi.fn().mockResolvedValue(undefined);+ const acquireLock = vi.fn().mockResolvedValue({ release });+ getMock.mockRejectedValueOnce(axiosError(404));++ await gcpCloudDnsInsertTxtRecord(connection, ZONE, RECORD, '"token-a"', {+ acquireLock+ } as unknown as TDnsRecordLockKeyStore);++ expect(acquireLock).toHaveBeenCalledWith(+ [`acme-dns-record-mutex-connection-id-${ZONE.toLowerCase()}-${FQDN}`],+ expect.any(Number),+ expect.anything()+ );+ expect(postMock).toHaveBeenCalledTimes(1);+ expect(release).toHaveBeenCalledTimes(1);+ });++ it("fails without touching the record when another order holds the lock", async () => {+ const acquireLock = vi.fn().mockRejectedValue(new Error("lock busy"));++ await expect(+ gcpCloudDnsInsertTxtRecord(connection, ZONE, RECORD, '"token-a"', {+ acquireLock+ } as unknown as TDnsRecordLockKeyStore)+ ).rejects.toThrow("Another certificate order is still using it.");+ expect(getMock).not.toHaveBeenCalled();+ expect(postMock).not.toHaveBeenCalled();+ });++ it("serializes concurrent updates to the same record in one process", async () => {+ let releaseFirstRead: (value: unknown) => void = () => {};+ getMock+ .mockImplementationOnce(+ () =>+ new Promise((resolve) => {+ releaseFirstRead = resolve;+ })+ )+ .mockResolvedValueOnce(existingRecordSet(['"token-a"']));++ const first = gcpCloudDnsInsertTxtRecord(connection, ZONE, RECORD, '"token-a"');+ const second = gcpCloudDnsInsertTxtRecord(connection, ZONE, RECORD, '"token-b"');+ await new Promise((resolve) => {+ setImmediate(resolve);+ });+ expect(getMock).toHaveBeenCalledTimes(1);++ releaseFirstRead(Promise.reject(axiosError(404)));+ await Promise.all([first, second]);++ expect(postMock).toHaveBeenCalledTimes(2);+ expect(postMock).toHaveBeenLastCalledWith(+ `${ZONE_URL}/changes`,+ expect.objectContaining({+ additions: [expect.objectContaining({ rrdatas: ['"token-a"', '"token-b"'] })]+ }),+ expect.anything()+ );+ });+it("retries as a fresh create when the record set was deleted between read and write", async () => {getMock.mockResolvedValueOnce(existingRecordSet(['"token-a"'])).mockRejectedValueOnce(axiosError(404));postMock.mockRejectedValueOnce(axiosError(404)).mockResolvedValueOnce({ data: {} });diff --git a/backend/src/services/certificate-authority/acme/dns-providers/gcp-cloud-dns.ts b/backend/src/services/certificate-authority/acme/dns-providers/gcp-cloud-dns.tsindex 2caeb7f8bf..d28d45622a 100644--- a/backend/src/services/certificate-authority/acme/dns-providers/gcp-cloud-dns.ts+++ b/backend/src/services/certificate-authority/acme/dns-providers/gcp-cloud-dns.ts@@ -11,6 +11,8 @@ import { getGcpConnectionAuthToken } from "@app/services/app-connection/gcp/gcp-import { TGcpConnection } from "@app/services/app-connection/gcp/gcp-connection-types";import { IntegrationUrls } from "@app/services/integration-auth/integration-list";+import { TDnsRecordLockKeyStore, withDnsRecordLock } from "./dns-record-lock";+type TGcpResourceRecordSet = {name: string;type: string;@@ -18,10 +20,15 @@ type TGcpResourceRecordSet = {rrdatas: string[];};+type TRecordSetChange = { additions?: TGcpResourceRecordSet[]; deletions?: TGcpResourceRecordSet[] };++type TBuildChange = (fqdn: string, existing: TGcpResourceRecordSet | null) => TRecordSetChange | null;+const TXT_RECORD_TTL_SECONDS = 60;const MAX_CHANGE_ATTEMPTS = 5;const CHANGE_RETRY_DELAY_MS = 1000;const REQUEST_TIMEOUT_MS = 30_000;+const RECORD_LOCK_TTL_MS = MAX_CHANGE_ATTEMPTS * (REQUEST_TIMEOUT_MS * 2 + CHANGE_RETRY_DELAY_MS) + 30_000;const QUOTES_REGEX = new RE2('"', "g");export const validateGcpCloudDnsZone = (hostedZoneId: string) => {@@ -84,28 +91,20 @@ const getTxtRecordSet = async (zoneUrl: string, accessToken: string, fqdn: strin}};-const submitChange = async (- zoneUrl: string,- accessToken: string,- change: { additions?: TGcpResourceRecordSet[]; deletions?: TGcpResourceRecordSet[] }-) => {+const submitChange = async (zoneUrl: string, accessToken: string, change: TRecordSetChange) => {await request.post(`${zoneUrl}/changes`, change, {headers: { Authorization: `Bearer ${accessToken}`, "Content-Type": "application/json", Accept: "application/json" },timeout: REQUEST_TIMEOUT_MS});};-const applyTxtRecordChange = async (+const submitWithRetry = async (connection: TGcpConnection,hostedZoneId: string,- recordName: string,- buildChange: (- fqdn: string,- existing: TGcpResourceRecordSet | null- ) => { additions?: TGcpResourceRecordSet[]; deletions?: TGcpResourceRecordSet[] } | null+ zoneUrl: string,+ fqdn: string,+ buildChange: TBuildChange) => {- const zoneUrl = getZoneUrl(hostedZoneId);- const fqdn = toFqdn(recordName);const accessToken = await getGcpConnectionAuthToken(connection);for (let attempt = 1; attempt <= MAX_CHANGE_ATTEMPTS; attempt += 1) {@@ -130,48 +129,85 @@ const applyTxtRecordChange = async (}};+const applyTxtRecordChange = async (+ connection: TGcpConnection,+ hostedZoneId: string,+ recordName: string,+ buildChange: TBuildChange,+ keyStore?: TDnsRecordLockKeyStore+) => {+ const zoneUrl = getZoneUrl(hostedZoneId);+ const fqdn = toFqdn(recordName);++ await withDnsRecordLock(+ {+ connectionId: connection.id,+ zoneId: hostedZoneId,+ name: fqdn,+ providerName: "Google Cloud DNS",+ lockTtlMs: RECORD_LOCK_TTL_MS+ },+ keyStore,+ () => submitWithRetry(connection, hostedZoneId, zoneUrl, fqdn, buildChange)+ );+};+export const gcpCloudDnsInsertTxtRecord = async (connection: TGcpConnection,hostedZoneId: string,recordName: string,- value: string+ value: string,+ keyStore?: TDnsRecordLockKeyStore) => {- await applyTxtRecordChange(connection, hostedZoneId, recordName, (fqdn, existing) => {- if (!existing) {- return { additions: [{ name: fqdn, type: "TXT", ttl: TXT_RECORD_TTL_SECONDS, rrdatas: [value] }] };- }+ await applyTxtRecordChange(+ connection,+ hostedZoneId,+ recordName,+ (fqdn, existing) => {+ if (!existing) {+ return { additions: [{ name: fqdn, type: "TXT", ttl: TXT_RECORD_TTL_SECONDS, rrdatas: [value] }] };+ }- if (existing.rrdatas.some((rrdata) => normalizeTxtValue(rrdata) === normalizeTxtValue(value))) {- return null;- }+ if (existing.rrdatas.some((rrdata) => normalizeTxtValue(rrdata) === normalizeTxtValue(value))) {+ return null;+ }- return {- deletions: [existing],- additions: [{ ...existing, ttl: TXT_RECORD_TTL_SECONDS, rrdatas: [...existing.rrdatas, value] }]- };- });+ return {+ deletions: [existing],+ additions: [{ ...existing, rrdatas: [...existing.rrdatas, value] }]+ };+ },+ keyStore+ );};export const gcpCloudDnsDeleteTxtRecord = async (connection: TGcpConnection,hostedZoneId: string,recordName: string,- value: string+ value: string,+ keyStore?: TDnsRecordLockKeyStore) => {- await applyTxtRecordChange(connection, hostedZoneId, recordName, (fqdn, existing) => {- const remaining = existing?.rrdatas.filter((rrdata) => normalizeTxtValue(rrdata) !== normalizeTxtValue(value));-- if (!existing || !remaining || remaining.length === existing.rrdatas.length) {- logger.warn(- { hostedZoneId, recordName: fqdn },- `Google Cloud DNS TXT record not found for deletion [hostedZoneId=${hostedZoneId}] [recordName=${fqdn}]`- );- return null;- }+ await applyTxtRecordChange(+ connection,+ hostedZoneId,+ recordName,+ (fqdn, existing) => {+ const remaining = existing?.rrdatas.filter((rrdata) => normalizeTxtValue(rrdata) !== normalizeTxtValue(value));++ if (!existing || !remaining || remaining.length === existing.rrdatas.length) {+ logger.warn(+ { hostedZoneId, recordName: fqdn },+ `Google Cloud DNS TXT record not found for deletion [hostedZoneId=${hostedZoneId}] [recordName=${fqdn}]`+ );+ return null;+ }- return {- deletions: [existing],- ...(remaining.length ? { additions: [{ ...existing, rrdatas: remaining }] } : {})- };- });+ return {+ deletions: [existing],+ ...(remaining.length ? { additions: [{ ...existing, rrdatas: remaining }] } : {})+ };+ },+ keyStore+ );};diff --git a/backend/src/services/certificate-authority/acme/dns-providers/powerdns.ts b/backend/src/services/certificate-authority/acme/dns-providers/powerdns.tsindex 96585e6951..7caa5205e9 100644--- a/backend/src/services/certificate-authority/acme/dns-providers/powerdns.ts+++ b/backend/src/services/certificate-authority/acme/dns-providers/powerdns.ts@@ -1,4 +1,3 @@-import { KeyStorePrefixes, TKeyStoreFactory } from "@app/keystore/keystore";import { logger } from "@app/lib/logger";import {getPowerDnsZoneRrset,@@ -11,67 +10,19 @@ import {TPowerDnsConnectionConfig} from "@app/services/app-connection/powerdns/powerdns-connection-types";+import { TDnsRecordLockKeyStore, withDnsRecordLock } from "./dns-record-lock";+const ACME_CHALLENGE_TTL_SECONDS = 60;const RRSET_LOCK_TTL_MS = POWERDNS_REQUEST_TIMEOUT_MS * 2 + 30_000;-const RRSET_LOCK_RETRY = { retryCount: 50, retryDelay: 2_000, retryJitter: 300 };export type TPowerDnsProviderDeps = TPowerDnsGatewayDeps & {- keyStore?: Pick<TKeyStoreFactory, "acquireLock">;+ keyStore?: TDnsRecordLockKeyStore;};const toFqdn = (name: string) => (name.endsWith(".") ? name : `${name}.`);const toConnectionConfig = (connection: TPowerDnsConnection) => connection as unknown as TPowerDnsConnectionConfig;-const pendingRrsetOperations = new Map<string, Promise<void>>();--const withLocalRrsetLock = async <T>(key: string, operation: () => Promise<T>): Promise<T> => {- const pending = pendingRrsetOperations.get(key) ?? Promise.resolve();- const result = pending.then(operation, operation);- const settled = result.then(- () => undefined,- () => undefined- );-- pendingRrsetOperations.set(key, settled);- void settled.then(() => {- if (pendingRrsetOperations.get(key) === settled) pendingRrsetOperations.delete(key);- });-- return result;-};--const withRrsetLock = async <T>(- { connectionId, zoneId, name }: { connectionId: string; zoneId: string; name: string },- keyStore: Pick<TKeyStoreFactory, "acquireLock"> | undefined,- operation: () => Promise<T>-): Promise<T> =>- withLocalRrsetLock(`${connectionId}|${zoneId}|${name}`, async () => {- if (!keyStore) return operation();-- const lock = await keyStore- .acquireLock(- [KeyStorePrefixes.AcmeDnsRecordLock(connectionId, zoneId, name)],- RRSET_LOCK_TTL_MS,- RRSET_LOCK_RETRY- )- .catch(() => null);-- if (!lock) {- throw new Error(- `Timed out waiting to update the PowerDNS record '${name}'. Another certificate order is still using it.`- );- }-- try {- return await operation();- } finally {- await lock.release().catch((error) => {- logger.warn(error, `Failed to release PowerDNS record lock [zoneId=${zoneId}] [name=${name}]`);- });- }- });-export const powerDnsInsertTxtRecord = async (connection: TPowerDnsConnection,hostedZoneId: string,@@ -86,29 +37,33 @@ export const powerDnsInsertTxtRecord = async (logger.info({ zoneId, name }, `Inserting TXT record for PowerDNS [zoneId=${zoneId}] [name=${name}]`);- await withRrsetLock({ connectionId: connection.id, zoneId, name }, keyStore, async () => {- const existing = await getPowerDnsZoneRrset(config, { zoneId, name, type: "TXT" }, gatewayDeps);- const existingRecords = existing?.records ?? [];-- if (existingRecords.some((record) => record.content === value)) return;-- await patchPowerDnsZoneRrsets(- config,- {- zoneId,- rrsets: [- {- name,- type: "TXT",- ttl: ACME_CHALLENGE_TTL_SECONDS,- changetype: "REPLACE",- records: [...existingRecords, { content: value, disabled: false }]- }- ]- },- gatewayDeps- );- });+ await withDnsRecordLock(+ { connectionId: connection.id, zoneId, name, providerName: "PowerDNS", lockTtlMs: RRSET_LOCK_TTL_MS },+ keyStore,+ async () => {+ const existing = await getPowerDnsZoneRrset(config, { zoneId, name, type: "TXT" }, gatewayDeps);+ const existingRecords = existing?.records ?? [];++ if (existingRecords.some((record) => record.content === value)) return;++ await patchPowerDnsZoneRrsets(+ config,+ {+ zoneId,+ rrsets: [+ {+ name,+ type: "TXT",+ ttl: ACME_CHALLENGE_TTL_SECONDS,+ changetype: "REPLACE",+ records: [...existingRecords, { content: value, disabled: false }]+ }+ ]+ },+ gatewayDeps+ );+ }+ );};export const powerDnsDeleteTxtRecord = async (@@ -125,33 +80,37 @@ export const powerDnsDeleteTxtRecord = async (logger.info({ zoneId, name }, `Deleting TXT record for PowerDNS [zoneId=${zoneId}] [name=${name}]`);- await withRrsetLock({ connectionId: connection.id, zoneId, name }, keyStore, async () => {- const existing = await getPowerDnsZoneRrset(config, { zoneId, name, type: "TXT" }, gatewayDeps);- const existingRecords = existing?.records ?? [];- const remainingRecords = existingRecords.filter((record) => record.content !== value);-- if (remainingRecords.length === existingRecords.length) {- logger.warn({ zoneId, name }, `PowerDNS TXT record to delete not found [zoneId=${zoneId}] [name=${name}]`);- return;+ await withDnsRecordLock(+ { connectionId: connection.id, zoneId, name, providerName: "PowerDNS", lockTtlMs: RRSET_LOCK_TTL_MS },+ keyStore,+ async () => {+ const existing = await getPowerDnsZoneRrset(config, { zoneId, name, type: "TXT" }, gatewayDeps);+ const existingRecords = existing?.records ?? [];+ const remainingRecords = existingRecords.filter((record) => record.content !== value);++ if (remainingRecords.length === existingRecords.length) {+ logger.warn({ zoneId, name }, `PowerDNS TXT record to delete not found [zoneId=${zoneId}] [name=${name}]`);+ return;+ }++ await patchPowerDnsZoneRrsets(+ config,+ {+ zoneId,+ rrsets: remainingRecords.length+ ? [+ {+ name,+ type: "TXT",+ ttl: existing?.ttl ?? ACME_CHALLENGE_TTL_SECONDS,+ changetype: "REPLACE",+ records: remainingRecords+ }+ ]+ : [{ name, type: "TXT", changetype: "DELETE" }]+ },+ gatewayDeps+ );}-- await patchPowerDnsZoneRrsets(- config,- {- zoneId,- rrsets: remainingRecords.length- ? [- {- name,- type: "TXT",- ttl: existing?.ttl ?? ACME_CHALLENGE_TTL_SECONDS,- changetype: "REPLACE",- records: remainingRecords- }- ]- : [{ name, type: "TXT", changetype: "DELETE" }]- },- gatewayDeps- );- });+ );};diff --git a/frontend/src/pages/cert-manager/CertificateAuthoritiesPage/components/ExternalCaFields/GcpCloudDnsZoneFields.tsx b/frontend/src/pages/cert-manager/CertificateAuthoritiesPage/components/ExternalCaFields/GcpCloudDnsZoneFields.tsxindex b67dbd1a55..a2b8af73bf 100644--- a/frontend/src/pages/cert-manager/CertificateAuthoritiesPage/components/ExternalCaFields/GcpCloudDnsZoneFields.tsx+++ b/frontend/src/pages/cert-manager/CertificateAuthoritiesPage/components/ExternalCaFields/GcpCloudDnsZoneFields.tsx@@ -154,7 +154,9 @@ export const GcpCloudDnsZoneFields = ({ control, connectionId }: Props) => {? "No matching projects": "No projects found. Grant the connection's service account the DNS Administrator and Service Usage Viewer roles on your project and enable the Cloud DNS API there."}- getOptionLabel={(option) => option.name}+ getOptionLabel={(option) =>+ option.name === option.id ? option.id : `${option.name} (${option.id})`+ }getOptionValue={(option) => option.id}isError={Boolean(projectsErrorMessage)}/>