Skip to content

Infisical is the open-source platform for securing secrets, certificates, access, and AI Agents.

bound UltraDNS record updates with timeouts and per-step retries

Carlos Monastyrski252cc1f14c8d
diff --git a/backend/src/services/app-connection/ultradns/ultradns-connection-fns.ts b/backend/src/services/app-connection/ultradns/ultradns-connection-fns.ts
index 5c71c0cdfa..267594de89 100644
--- a/backend/src/services/app-connection/ultradns/ultradns-connection-fns.ts
+++ b/backend/src/services/app-connection/ultradns/ultradns-connection-fns.ts
@@ -1,4 +1,4 @@
-import { AxiosError, isAxiosError } from "axios";
+import { AxiosError, AxiosInstance, isAxiosError } from "axios";
import { createRequestClient } from "@app/lib/config/request";
import { BadRequestError } from "@app/lib/errors";
@@ -19,7 +19,12 @@ const MAX_ZONE_PAGES = 100;
export const ULTRADNS_REQUEST_TIMEOUT_MS = 10_000;
-export const ultraDNSRequest = createRequestClient({ timeout: ULTRADNS_REQUEST_TIMEOUT_MS }, { retries: 0 });
+export const ultraDNSRequest = createRequestClient({ timeout: ULTRADNS_REQUEST_TIMEOUT_MS });
+
+export const ultraDNSSingleAttemptRequest = createRequestClient(
+ { timeout: ULTRADNS_REQUEST_TIMEOUT_MS },
+ { retries: 0 }
+);
export const getUltraDNSUrl = (environment: UltraDNSEnvironment, path: string) => {
const baseUrl =
@@ -39,8 +44,13 @@ export const getUltraDNSErrorMessage = (error: unknown) => {
return error instanceof Error ? error.message : "Unknown error";
};
-export const getUltraDNSAccessToken = async (environment: UltraDNSEnvironment, username: string, password: string) => {
- const { data } = await ultraDNSRequest.post<{ accessToken: string }>(
+export const getUltraDNSAccessToken = async (
+ environment: UltraDNSEnvironment,
+ username: string,
+ password: string,
+ client: AxiosInstance = ultraDNSRequest
+) => {
+ const { data } = await client.post<{ accessToken: string }>(
getUltraDNSUrl(environment, "/v1/authorization/token"),
new URLSearchParams({ grant_type: "password", username, password }).toString(),
{
diff --git a/backend/src/services/certificate-authority/acme/dns-providers/dns-record-lock.ts b/backend/src/services/certificate-authority/acme/dns-providers/dns-record-lock.ts
index cf6ce258b8..4f4d8497ee 100644
--- a/backend/src/services/certificate-authority/acme/dns-providers/dns-record-lock.ts
+++ b/backend/src/services/certificate-authority/acme/dns-providers/dns-record-lock.ts
@@ -1,7 +1,8 @@
import { KeyStorePrefixes, TKeyStoreFactory } from "@app/keystore/keystore";
import { logger } from "@app/lib/logger";
-const DNS_RECORD_LOCK_RETRY = { retryCount: 50, retryDelay: 2_000, retryJitter: 300 };
+const DNS_RECORD_LOCK_RETRY_DELAY_MS = 2_000;
+const DNS_RECORD_LOCK_RETRY_JITTER_MS = 300;
export const withDnsRecordLock = async <T>(
{
@@ -15,7 +16,11 @@ export const withDnsRecordLock = async <T>(
operation: () => Promise<T>
): Promise<T> => {
const lock = await keyStore
- .acquireLock([KeyStorePrefixes.AcmeDnsRecordLock(connectionId, zoneId, name)], lockTtlMs, DNS_RECORD_LOCK_RETRY)
+ .acquireLock([KeyStorePrefixes.AcmeDnsRecordLock(connectionId, zoneId, name)], lockTtlMs, {
+ retryCount: Math.ceil(lockTtlMs / DNS_RECORD_LOCK_RETRY_DELAY_MS) + 5,
+ retryDelay: DNS_RECORD_LOCK_RETRY_DELAY_MS,
+ retryJitter: DNS_RECORD_LOCK_RETRY_JITTER_MS
+ })
.catch(() => null);
if (!lock) {
diff --git a/backend/src/services/certificate-authority/acme/dns-providers/ultradns.ts b/backend/src/services/certificate-authority/acme/dns-providers/ultradns.ts
index 99f93eb8c9..a022ae1b02 100644
--- a/backend/src/services/certificate-authority/acme/dns-providers/ultradns.ts
+++ b/backend/src/services/certificate-authority/acme/dns-providers/ultradns.ts
@@ -11,18 +11,19 @@ import {
getUltraDNSErrorMessage,
getUltraDNSUrl,
ULTRADNS_REQUEST_TIMEOUT_MS,
- ultraDNSRequest
+ ultraDNSSingleAttemptRequest
} from "@app/services/app-connection/ultradns/ultradns-connection-fns";
import { TUltraDNSConnection } from "@app/services/app-connection/ultradns/ultradns-connection-types";
import { withDnsRecordLock } from "./dns-record-lock";
const TXT_RECORD_TTL = 60;
-const MAX_WRITE_ATTEMPTS = 3;
-const WRITE_RETRY_DELAY_MS = 3000;
+const MAX_ATTEMPTS = 3;
+const RETRY_DELAY_MS = 3000;
const APPLY_PENDING_STATUS = 202;
-const NON_RETRYABLE_WRITE_STATUSES = [401, 403, 404];
-const RECORD_LOCK_TTL_MS = ULTRADNS_REQUEST_TIMEOUT_MS * (2 + MAX_WRITE_ATTEMPTS * 2) + 30_000;
+const NON_RETRYABLE_STATUSES = [401, 403, 404];
+const REQUESTS_PER_ATTEMPT = 3;
+const RECORD_LOCK_TTL_MS = ULTRADNS_REQUEST_TIMEOUT_MS * (MAX_ATTEMPTS * REQUESTS_PER_ATTEMPT + 1) + 30_000;
export type TUltraDNSProviderDeps = {
keyStore: Pick<TKeyStoreFactory, "acquireLock">;
@@ -42,7 +43,7 @@ const getTxtRrSetUrl = (environment: UltraDNSEnvironment, zoneName: string, reco
const getTxtRecordValues = async (url: string, accessToken: string) => {
try {
- const { data } = await ultraDNSRequest.get<{ rrSets?: { rdata?: string[] }[] }>(url, {
+ const { data } = await ultraDNSSingleAttemptRequest.get<{ rrSets?: { rdata?: string[] }[] }>(url, {
headers: {
Authorization: `Bearer ${accessToken}`,
Accept: "application/json"
@@ -64,49 +65,59 @@ const writeTxtRecordValues = async (url: string, accessToken: string, values: st
};
if (!values.length) {
- const { status } = await ultraDNSRequest.delete(url, { headers });
+ const { status } = await ultraDNSSingleAttemptRequest.delete(url, { headers });
return status;
}
const body = { ttl: TXT_RECORD_TTL, rdata: values };
const { status } = await (isNewRrSet
- ? ultraDNSRequest.post(url, body, { headers })
- : ultraDNSRequest.put(url, body, { headers }));
+ ? ultraDNSSingleAttemptRequest.post(url, body, { headers })
+ : ultraDNSSingleAttemptRequest.put(url, body, { headers }));
return status;
};
+const toError = (error: unknown) => (error instanceof Error ? error : new Error(String(error)));
+
+const isNonRetryableError = (error: unknown) =>
+ isAxiosError(error) && NON_RETRYABLE_STATUSES.includes(error.response?.status ?? 0);
+
const applyToTxtRecordSet = async (
+ connection: TUltraDNSConnection,
url: string,
- accessToken: string,
nextValuesFor: (currentValues: string[]) => string[] | null
) => {
- let lastWriteError: Error | undefined;
- let hasWritten = false;
+ const { username, password, environment } = connection.credentials;
+ let accessToken: string | undefined;
+ let lastError: Error | undefined;
let shouldWaitBeforeRetry = false;
- for (let attempt = 1; attempt <= MAX_WRITE_ATTEMPTS; attempt += 1) {
- if (shouldWaitBeforeRetry) await delay(WRITE_RETRY_DELAY_MS * 2 ** (attempt - 2));
+ for (let attempt = 1; attempt <= MAX_ATTEMPTS; attempt += 1) {
+ if (shouldWaitBeforeRetry) await delay(RETRY_DELAY_MS * 2 ** (attempt - 2));
- const currentValues = await getTxtRecordValues(url, accessToken);
- const nextValues = nextValuesFor(currentValues);
+ try {
+ accessToken ??= await getUltraDNSAccessToken(environment, username, password, ultraDNSSingleAttemptRequest);
+ const currentValues = await getTxtRecordValues(url, accessToken);
+ const nextValues = nextValuesFor(currentValues);
- if (!nextValues) return { isConfirmed: true, lastWriteError };
+ if (!nextValues) return { isConfirmed: true, lastError };
- try {
const status = await writeTxtRecordValues(url, accessToken, nextValues, currentValues.length === 0);
- hasWritten = true;
shouldWaitBeforeRetry = status === APPLY_PENDING_STATUS;
} catch (error) {
- if (isAxiosError(error) && NON_RETRYABLE_WRITE_STATUSES.includes(error.response?.status ?? 0)) throw error;
- lastWriteError = error instanceof Error ? error : new Error(String(error));
+ if (isNonRetryableError(error)) throw error;
+ lastError = toError(error);
shouldWaitBeforeRetry = true;
}
}
- if (!hasWritten && lastWriteError) throw lastWriteError;
+ if (!accessToken) return { isConfirmed: false, lastError };
- const finalValues = await getTxtRecordValues(url, accessToken);
- return { isConfirmed: !nextValuesFor(finalValues), lastWriteError };
+ try {
+ const finalValues = await getTxtRecordValues(url, accessToken);
+ return { isConfirmed: !nextValuesFor(finalValues), lastError };
+ } catch (error) {
+ return { isConfirmed: false, lastError: toError(error) };
+ }
};
export const ultraDNSInsertTxtRecord = async (
@@ -116,9 +127,7 @@ export const ultraDNSInsertTxtRecord = async (
value: string,
{ keyStore }: TUltraDNSProviderDeps
) => {
- const {
- credentials: { username, password, environment }
- } = connection;
+ const { environment } = connection.credentials;
const zoneId = toFullyQualifiedName(zoneName);
const name = toFullyQualifiedName(recordName);
@@ -127,18 +136,17 @@ export const ultraDNSInsertTxtRecord = async (
{ providerName: "UltraDNS", connectionId: connection.id, zoneId, name, lockTtlMs: RECORD_LOCK_TTL_MS },
keyStore,
async () => {
- const accessToken = await getUltraDNSAccessToken(environment, username, password);
- const { isConfirmed, lastWriteError } = await applyToTxtRecordSet(
+ const { isConfirmed, lastError } = await applyToTxtRecordSet(
+ connection,
getTxtRrSetUrl(environment, zoneId, name),
- accessToken,
(currentValues) =>
currentValues.some((current) => unquote(current) === unquote(value)) ? null : [...currentValues, value]
);
if (!isConfirmed) {
throw new Error(
- `UltraDNS did not confirm the challenge record after ${MAX_WRITE_ATTEMPTS} attempts${
- lastWriteError ? `: ${getUltraDNSErrorMessage(lastWriteError)}` : ""
+ `UltraDNS did not confirm the challenge record after ${MAX_ATTEMPTS} attempts${
+ lastError ? `: ${getUltraDNSErrorMessage(lastError)}` : ""
}`
);
}
@@ -159,9 +167,7 @@ export const ultraDNSDeleteTxtRecord = async (
value: string,
{ keyStore }: TUltraDNSProviderDeps
) => {
- const {
- credentials: { username, password, environment }
- } = connection;
+ const { environment } = connection.credentials;
const zoneId = toFullyQualifiedName(zoneName);
const name = toFullyQualifiedName(recordName);
@@ -170,10 +176,9 @@ export const ultraDNSDeleteTxtRecord = async (
{ providerName: "UltraDNS", connectionId: connection.id, zoneId, name, lockTtlMs: RECORD_LOCK_TTL_MS },
keyStore,
async () => {
- const accessToken = await getUltraDNSAccessToken(environment, username, password);
- const { isConfirmed, lastWriteError } = await applyToTxtRecordSet(
+ const { isConfirmed, lastError } = await applyToTxtRecordSet(
+ connection,
getTxtRrSetUrl(environment, zoneId, name),
- accessToken,
(currentValues) => {
const remainingValues = currentValues.filter((current) => unquote(current) !== unquote(value));
return remainingValues.length === currentValues.length ? null : remainingValues;
@@ -182,7 +187,7 @@ export const ultraDNSDeleteTxtRecord = async (
if (!isConfirmed) {
logger.warn(
- { zoneName: zoneId, recordName: name, err: lastWriteError },
+ { zoneName: zoneId, recordName: name, err: lastError },
"Could not remove the UltraDNS TXT record for the ACME challenge"
);
}