Skip to content

Infisical is the open-source platform for securing secrets, certificates, access, and AI Agents.

fix(pki): stop retrying refused imperva uploads and claim slots on success

bernie-g155ae41d5217
diff --git a/backend/src/services/pki-sync/imperva/imperva-pki-sync-fns.test.ts b/backend/src/services/pki-sync/imperva/imperva-pki-sync-fns.test.ts
index 798f34429c..d659422a3d 100644
--- a/backend/src/services/pki-sync/imperva/imperva-pki-sync-fns.test.ts
+++ b/backend/src/services/pki-sync/imperva/imperva-pki-sync-fns.test.ts
@@ -111,7 +111,7 @@ describe("Imperva upload failures", () => {
const rejected = new ImpervaRequestError({ message: "Invalid input", status: 400 });
expect(describeImpervaUploadFailure(rejected)).toBe(
- "Invalid input. Imperva rejected the certificate. It has to cover every domain the website serves, and its chain has to be complete and ordered leaf first"
+ "Invalid input. Imperva rejected the certificate. It has to cover every domain the website serves, and its key has to be RSA under 4096 bits or ECDSA P-256"
);
});
@@ -361,3 +361,55 @@ describe("Imperva deployment mismatch", () => {
expect(updateById.mock.calls[0][1]).toMatchObject({ syncStatus: "failed" });
});
});
+
+describe("Imperva retry decisions", () => {
+ test("stops retrying what Imperva refused, keeps retrying what it could not answer", async () => {
+ const { isImpervaRetryable } = await import("./imperva-pki-sync-fns");
+ const { ImpervaRequestError } = await import("@app/services/app-connection/imperva/imperva-connection-fns");
+
+ expect(isImpervaRetryable(new ImpervaRequestError({ message: "Invalid input", status: 400 }))).toBe(false);
+ expect(isImpervaRetryable(new ImpervaRequestError({ message: "unauthorized", status: 401 }))).toBe(false);
+ expect(isImpervaRetryable(new ImpervaRequestError({ message: "rate limited", status: 429 }))).toBe(true);
+ expect(isImpervaRetryable(new ImpervaRequestError({ message: "server error", status: 500 }))).toBe(true);
+ expect(isImpervaRetryable(new Error("socket hang up"))).toBe(true);
+ });
+
+ test("does not claim a slot when the upload is rejected", async () => {
+ const { impervaPkiSyncFactory } = await import("./imperva-pki-sync-fns");
+ const { ImpervaRequestError } = await import("@app/services/app-connection/imperva/imperva-connection-fns");
+
+ impervaRequest.mockReset();
+ impervaRequest.mockRejectedValue(new ImpervaRequestError({ message: "Invalid input", status: 400 }));
+
+ const claimExternalIdentifier = vi.fn();
+ const factory = impervaPkiSyncFactory({
+ certificateSyncDAL: {
+ findByPkiSyncId: vi.fn().mockResolvedValue([]),
+ removeCertificates: vi.fn(),
+ addCertificates: vi.fn(),
+ findByPkiSyncAndCertificate: vi.fn().mockResolvedValue(undefined),
+ updateById: vi.fn(),
+ updateSyncStatus: vi.fn(),
+ findExternalIdentifiersInUse: vi.fn().mockResolvedValue(new Set<string>()),
+ claimExternalIdentifier
+ } as never,
+ certificateDAL: { findById: vi.fn().mockResolvedValue(undefined) } as never
+ });
+
+ await expect(
+ factory.syncCertificates(
+ {
+ id: "sync-1",
+ destination: "imperva",
+ projectId: "project-1",
+ destinationConfig: { siteId: "111", siteName: "waf.example.com" },
+ syncOptions: { canRemoveCertificates: true },
+ connection: { credentials: { apiId: "id", apiKey: "key" }, gatewayId: null, gatewayPoolId: null }
+ } as never,
+ { leaf: { cert: RSA_CERT_PEM, privateKey: "key", certificateId: "cert-1" } } as never
+ )
+ ).rejects.toMatchObject({ shouldRetry: false });
+
+ expect(claimExternalIdentifier).not.toHaveBeenCalled();
+ });
+});
diff --git a/backend/src/services/pki-sync/imperva/imperva-pki-sync-fns.ts b/backend/src/services/pki-sync/imperva/imperva-pki-sync-fns.ts
index 661b42fbb5..df1995019d 100644
--- a/backend/src/services/pki-sync/imperva/imperva-pki-sync-fns.ts
+++ b/backend/src/services/pki-sync/imperva/imperva-pki-sync-fns.ts
@@ -23,6 +23,7 @@ import { CertificateSyncStatus } from "@app/services/certificate-sync/certificat
import { TCertificateMap } from "@app/services/pki-sync/pki-sync-types";
import { PkiSyncError } from "../pki-sync-errors";
+import { resolveEffectiveSyncGatewayId } from "../pki-sync-gateway-fns";
import { TPkiSyncWithCredentials } from "../pki-sync-types";
import { ImpervaCertificateAuthType } from "./imperva-pki-sync-constants";
import { TImpervaPkiSyncConfig } from "./imperva-pki-sync-types";
@@ -56,7 +57,13 @@ const IMPERVA_CUSTOM_CERT_TYPE = "CUSTOM_CERT";
// Imperva answers a rejected upload with a bare "Invalid input", which tells the user nothing about
// what to change, so the conditions it enforces are spelled out alongside its message.
const IMPERVA_UPLOAD_REJECTION_HINT =
- "Imperva rejected the certificate. It has to cover every domain the website serves, and its chain has to be complete and ordered leaf first";
+ "Imperva rejected the certificate. It has to cover every domain the website serves, and its key has to be RSA under 4096 bits or ECDSA P-256";
+
+// 429 aside, a 4xx is Imperva refusing the request itself, so repeating it changes nothing.
+export const isImpervaRetryable = (error: unknown): boolean => {
+ if (!(error instanceof ImpervaRequestError) || error.status === undefined) return true;
+ return error.status === 429 || error.status < 400 || error.status >= 500;
+};
export const describeImpervaUploadFailure = (error: unknown): string => {
const message = extractImpervaErrorMessage(error, "Unknown error");
@@ -120,15 +127,6 @@ export const impervaPkiSyncFactory = ({
gatewayV2Service,
gatewayPoolService
}: TImpervaPkiSyncFactoryDeps) => {
- const resolveGateway = async (pkiSync: TPkiSyncWithCredentials) => {
- return gatewayPoolService
- ? gatewayPoolService.resolveEffectiveGatewayId({
- gatewayId: pkiSync.connection.gatewayId,
- gatewayPoolId: pkiSync.connection.gatewayPoolId
- })
- : (pkiSync.connection.gatewayId ?? null);
- };
-
const uploadCustomCertificate = async (
makeRequest: TImpervaRequestFn,
credentials: TImpervaCredentials,
@@ -266,6 +264,75 @@ export const impervaPkiSyncFactory = ({
return { certificate: selected, skippedCertificates };
};
+ // Clears the slots this run did not write to, on whichever website each one belongs to.
+ const removeOrphanedSlots = async (args: {
+ pkiSync: TPkiSyncWithCredentials;
+ makeRequest: TImpervaRequestFn;
+ credentials: TImpervaCredentials;
+ siteId: string;
+ siteLabel: string;
+ existingSyncRecords: TCertificateSyncs[];
+ occupiedSlots: Set<string>;
+ activeCertificateIds: Set<string>;
+ }): Promise<{ removed: number; failedRemovals: Array<{ name: string; error: string }> }> => {
+ const { pkiSync, makeRequest, credentials, siteId, siteLabel, existingSyncRecords } = args;
+ const { occupiedSlots, activeCertificateIds } = args;
+ const failedRemovals: Array<{ name: string; error: string }> = [];
+ let removed = 0;
+
+ // Keyed on the slot rather than the certificate, so repointing the sync to another website
+ // still clears the slot the certificate used to occupy on the old one.
+ const orphanedRecords = existingSyncRecords.filter(
+ (record: TCertificateSyncs) => record.externalIdentifier && !occupiedSlots.has(record.externalIdentifier)
+ );
+
+ const ownedByOtherSync = await certificateSyncDAL.findExternalIdentifiersInUse(
+ orphanedRecords.map((record: TCertificateSyncs) => record.externalIdentifier as string),
+ { excludePkiSyncId: pkiSync.id, destination: pkiSync.destination }
+ );
+
+ for (const record of orphanedRecords) {
+ if (ownedByOtherSync.has(record.externalIdentifier as string)) {
+ if (record.certificateId && !activeCertificateIds.has(record.certificateId)) {
+ await certificateSyncDAL.removeCertificates(pkiSync.id, [record.certificateId]);
+ }
+ // eslint-disable-next-line no-continue
+ continue;
+ }
+
+ // The slot a record occupies belongs to the site it was uploaded to, which is not the sync's
+ // current site once the destination has been repointed.
+ const { siteId: recordSiteId, authType } = parseImpervaExternalIdentifier(record.externalIdentifier as string);
+ const recordSiteLabel = recordSiteId === siteId ? siteLabel : recordSiteId;
+
+ try {
+ await deleteCustomCertificate(makeRequest, credentials, recordSiteId, authType);
+ removed += 1;
+
+ // The row survives when the same certificate was just written to a different slot, since
+ // it now points at that new slot.
+ if (record.certificateId && !activeCertificateIds.has(record.certificateId)) {
+ await certificateSyncDAL.removeCertificates(pkiSync.id, [record.certificateId]);
+ }
+
+ logger.info(
+ `Imperva PKI sync [syncId=${pkiSync.id}]: removed orphaned ${authType} custom certificate from site ${recordSiteLabel}`
+ );
+ } catch (error: unknown) {
+ failedRemovals.push({
+ name: record.externalIdentifier as string,
+ error: extractImpervaErrorMessage(error, "Unknown error")
+ });
+ logger.error(
+ { error },
+ `Imperva PKI sync [syncId=${pkiSync.id}]: failed to remove ${authType} custom certificate from site ${recordSiteLabel}`
+ );
+ }
+ }
+
+ return { removed, failedRemovals };
+ };
+
const syncCertificates = async (
pkiSync: TPkiSyncWithCredentials,
certificateMap: TCertificateMap
@@ -287,7 +354,7 @@ export const impervaPkiSyncFactory = ({
const canRemoveCertificates = syncOptions?.canRemoveCertificates ?? true;
const siteLabel = siteName ? `${siteName} (${siteId})` : siteId;
- const effectiveGatewayId = await resolveGateway(pkiSync);
+ const effectiveGatewayId = await resolveEffectiveSyncGatewayId(pkiSync, gatewayPoolService);
return withImpervaOperation(
{ gatewayId: effectiveGatewayId, credentials },
@@ -299,6 +366,7 @@ export const impervaPkiSyncFactory = ({
const failedUploads: Array<{ name: string; error: string }> = [];
// Uploaded but not served. Reported per certificate so its row does not read Succeeded.
const undeployedCertificates: Array<{ name: string; error: string }> = [];
+ let uploadRetryable = true;
const failedRemovals: Array<{ name: string; error: string }> = [];
const occupiedSlots = new Set<string>();
let deploymentWarning = "";
@@ -322,16 +390,6 @@ export const impervaPkiSyncFactory = ({
const externalIdentifier = buildImpervaExternalIdentifier(siteId, authType);
try {
- // Claimed before the upload so a second sync pointed at the same slot cannot take it over
- // while this one is writing to it.
- if (certificate.certificateId) {
- await certificateSyncDAL.claimExternalIdentifier(
- pkiSync.id,
- certificate.certificateId,
- externalIdentifier
- );
- }
-
await uploadCustomCertificate(makeRequest, credentials, siteId, {
certificate: buildImpervaCertificatePayload(certificate.cert, certificate.certificateChain),
privateKey: Buffer.from(certificate.privateKey.trim()).toString("base64"),
@@ -341,6 +399,16 @@ export const impervaPkiSyncFactory = ({
occupiedSlots.add(externalIdentifier);
uploaded += 1;
+ // Claimed only once the upload has landed. A claim for a slot nothing was written to
+ // would stop another sync cleaning up a certificate this one never placed.
+ if (certificate.certificateId) {
+ await certificateSyncDAL.claimExternalIdentifier(
+ pkiSync.id,
+ certificate.certificateId,
+ externalIdentifier
+ );
+ }
+
deploymentWarning = await buildDeploymentWarning(makeRequest, credentials, siteId, authType);
if (deploymentWarning) {
undeployedCertificates.push({ name: certificate.certName, error: deploymentWarning });
@@ -379,6 +447,7 @@ export const impervaPkiSyncFactory = ({
);
} catch (error: unknown) {
const errorMessage = describeImpervaUploadFailure(error);
+ uploadRetryable = isImpervaRetryable(error);
failedUploads.push({ name: certificate.certName, error: errorMessage });
if (certificate.certificateId) {
@@ -404,57 +473,18 @@ export const impervaPkiSyncFactory = ({
// A failed upload leaves the previous certificate serving, and clearing its slot would
// take the website off TLS.
if (canRemoveCertificates && failedUploads.length === 0) {
- // Keyed on the slot rather than the certificate, so repointing the sync to another website
- // still clears the slot the certificate used to occupy on the old one.
- const orphanedRecords = existingSyncRecords.filter(
- (record: TCertificateSyncs) => record.externalIdentifier && !occupiedSlots.has(record.externalIdentifier)
- );
-
- const ownedByOtherSync = await certificateSyncDAL.findExternalIdentifiersInUse(
- orphanedRecords.map((record: TCertificateSyncs) => record.externalIdentifier as string),
- { excludePkiSyncId: pkiSync.id, destination: pkiSync.destination }
- );
-
- for (const record of orphanedRecords) {
- if (ownedByOtherSync.has(record.externalIdentifier as string)) {
- if (record.certificateId && !activeCertificateIds.has(record.certificateId)) {
- await certificateSyncDAL.removeCertificates(pkiSync.id, [record.certificateId]);
- }
- // eslint-disable-next-line no-continue
- continue;
- }
-
- // The slot a record occupies belongs to the site it was uploaded to, which is not the sync's
- // current site once the destination has been repointed.
- const { siteId: recordSiteId, authType } = parseImpervaExternalIdentifier(
- record.externalIdentifier as string
- );
- const recordSiteLabel = recordSiteId === siteId ? siteLabel : recordSiteId;
-
- try {
- await deleteCustomCertificate(makeRequest, credentials, recordSiteId, authType);
- removed += 1;
-
- // The row survives when the same certificate was just written to a different slot, since
- // it now points at that new slot.
- if (record.certificateId && !activeCertificateIds.has(record.certificateId)) {
- await certificateSyncDAL.removeCertificates(pkiSync.id, [record.certificateId]);
- }
-
- logger.info(
- `Imperva PKI sync [syncId=${pkiSync.id}]: removed orphaned ${authType} custom certificate from site ${recordSiteLabel}`
- );
- } catch (error: unknown) {
- failedRemovals.push({
- name: record.externalIdentifier as string,
- error: extractImpervaErrorMessage(error, "Unknown error")
- });
- logger.error(
- { error },
- `Imperva PKI sync [syncId=${pkiSync.id}]: failed to remove ${authType} custom certificate from site ${recordSiteLabel}`
- );
- }
- }
+ const cleanup = await removeOrphanedSlots({
+ pkiSync,
+ makeRequest,
+ credentials,
+ siteId,
+ siteLabel,
+ existingSyncRecords,
+ occupiedSlots,
+ activeCertificateIds
+ });
+ removed = cleanup.removed;
+ failedRemovals.push(...cleanup.failedRemovals);
}
// Only once the cleanup above has had its look at the old slot.
@@ -464,7 +494,8 @@ export const impervaPkiSyncFactory = ({
if (failedUploads.length > 0) {
throw new PkiSyncError({
- message: `Failed to sync certificate to Imperva site ${siteLabel}: ${failedUploads[0].error}`
+ message: `Failed to sync certificate to Imperva site ${siteLabel}: ${failedUploads[0].error}`,
+ shouldRetry: uploadRetryable
});
}
@@ -551,7 +582,7 @@ export const impervaPkiSyncFactory = ({
}
}
- const effectiveGatewayId = await resolveGateway(pkiSync);
+ const effectiveGatewayId = await resolveEffectiveSyncGatewayId(pkiSync, gatewayPoolService);
try {
await withImpervaOperation(
@@ -575,7 +606,8 @@ export const impervaPkiSyncFactory = ({
// is gone or the key lost access, and the certificate may still be installed.
if (error instanceof ImpervaRequestError && error.status === 404) {
throw new PkiSyncError({
- message: `Imperva website ${recordSiteLabel} is unknown to this connection, so its ${authType} custom certificate was left in place. Check that the website still exists and that the API key can reach its account.`
+ message: `Imperva website ${recordSiteLabel} is unknown to this connection, so its ${authType} custom certificate was left in place. Check that the website still exists and that the API key can reach its account.`,
+ shouldRetry: false
});
}
@@ -584,7 +616,7 @@ export const impervaPkiSyncFactory = ({
error,
"Unknown error"
)}`,
- shouldRetry: true
+ shouldRetry: isImpervaRetryable(error)
});
}
}
diff --git a/backend/src/services/pki-sync/pki-sync-gateway-fns.ts b/backend/src/services/pki-sync/pki-sync-gateway-fns.ts
new file mode 100644
index 0000000000..45e8aed3fa
--- /dev/null
+++ b/backend/src/services/pki-sync/pki-sync-gateway-fns.ts
@@ -0,0 +1,18 @@
+import { TGatewayPoolServiceFactory } from "@app/ee/services/gateway-pool/gateway-pool-service";
+
+type TGatewayBinding = {
+ connection: { gatewayId?: string | null; gatewayPoolId?: string | null };
+};
+
+// The gateway named on the connection, or whichever one its pool hands out.
+export const resolveEffectiveSyncGatewayId = async (
+ pkiSync: TGatewayBinding,
+ gatewayPoolService?: Pick<TGatewayPoolServiceFactory, "resolveEffectiveGatewayId">
+): Promise<string | null> => {
+ if (!gatewayPoolService) return pkiSync.connection.gatewayId ?? null;
+
+ return gatewayPoolService.resolveEffectiveGatewayId({
+ gatewayId: pkiSync.connection.gatewayId,
+ gatewayPoolId: pkiSync.connection.gatewayPoolId
+ });
+};