Skip to content

Infisical is the open-source platform for securing secrets, certificates, access, and AI Agents.

34.1 KB
Raw Blame
/* eslint-disable no-await-in-loop */
import { AxiosError } from "axios";
import { Job } from "bullmq";
import { randomUUID } from "crypto";

import { EventType, TAuditLogServiceFactory } from "@app/ee/services/audit-log/audit-log-types";
import { TGatewayPoolServiceFactory } from "@app/ee/services/gateway-pool/gateway-pool-service";
import { TGatewayV2ServiceFactory } from "@app/ee/services/gateway-v2/gateway-v2-service";
import { KeyStorePrefixes, TKeyStoreFactory, TKeyStoreLock } from "@app/keystore/keystore";
import { getConfig } from "@app/lib/config/env";
import { logger } from "@app/lib/logger";
import { highCardinalityMeter } from "@app/lib/telemetry/metrics";
import { QueueJobs, QueueName, TQueueServiceFactory } from "@app/queue";
import { ActorType } from "@app/services/auth/auth-type";
import { TKmsServiceFactory } from "@app/services/kms/kms-service";
import { TNotificationServiceFactory } from "@app/services/notification/notification-service";
import { TOidcIssuerServiceFactory } from "@app/services/oidc-issuer/oidc-issuer-service";
import { TPkiApplicationDALFactory } from "@app/services/pki-application/pki-application-dal";
import { hydratePkiSyncCredentials } from "@app/services/pki-sync/pki-sync-credentials-fns";
import { TProjectDALFactory } from "@app/services/project/project-dal";
import { TTelemetryServiceFactory } from "@app/services/telemetry/telemetry-service";
import { PostHogEventTypes } from "@app/services/telemetry/telemetry-types";

import {
  releaseAppConnectionConcurrency,
  tryAdmitAppConnectionConcurrency
} from "../app-connection/app-connection-concurrency-fns";
import { TAppConnectionDALFactory } from "../app-connection/app-connection-dal";
import { TCertificateAuthorityCertDALFactory } from "../certificate-authority/certificate-authority-cert-dal";
import { TCertificateAuthorityDALFactory } from "../certificate-authority/certificate-authority-dal";
import { TCertificateSyncDALFactory } from "../certificate-sync/certificate-sync-dal";
import { CertificateSyncStatus } from "../certificate-sync/certificate-sync-enums";
import { TCertificateBodyDALFactory } from "../certificate/certificate-body-dal";
import { TCertificateDALFactory } from "../certificate/certificate-dal";
import { TCertificateSecretDALFactory } from "../certificate/certificate-secret-dal";
import { buildCertificateMap } from "./pki-sync-certificate-map-fns";
import { TPkiSyncDALFactory } from "./pki-sync-dal";
import { PKI_SYNC_CONNECTION_LOCK_RETRY, PkiSyncFailureKind, PkiSyncStatus } from "./pki-sync-enums";
import { PkiSyncError } from "./pki-sync-errors";
import { notifyPkiSyncFailure } from "./pki-sync-failure-notification-fns";
import { withPkiSyncFilterLock } from "./pki-sync-filter-reconcile-fns";
import {
  getPkiSyncProviderCapabilities,
  isPkiSyncSerialisedPerConnection,
  parsePkiSyncErrorMessage,
  PkiSyncFns,
  truncateSyncMessage
} from "./pki-sync-fns";
import {
  buildHealthCheckCommandFailureMessage,
  didHealthCheckFail,
  getHealthCheckCommand,
  THealthCheckCommandResult
} from "./pki-sync-health-check-command-fns";
import {
  buildPostSyncCommandFailureMessage,
  getPostSyncCommand,
  TPostSyncCommandResult
} from "./pki-sync-post-sync-command-fns";
import { getPkiSyncTargetHost } from "./pki-sync-target-host-fns";
import {
  TCertificateMap,
  TPkiSyncImportCertificatesDTO,
  TPkiSyncRaw,
  TPkiSyncRemoveCertificatesDTO,
  TPkiSyncSyncCertificatesDTO,
  TQueuePkiSyncImportCertificatesByIdDTO,
  TQueuePkiSyncLinkMatchingCertificatesDTO,
  TQueuePkiSyncReconcileFiltersDTO,
  TQueuePkiSyncRemoveCertificatesByIdDTO,
  TQueuePkiSyncSyncCertificatesByIdDTO
} from "./pki-sync-types";
import { reconcileCertificateAgainstMatchingSyncs, reconcileSyncFilters } from "./pki-sync-utils";

export type TPkiSyncQueueFactory = ReturnType<typeof pkiSyncQueueFactory>;

type TPkiSyncQueueFactoryDep = {
  queueService: Pick<TQueueServiceFactory, "queue" | "start">;
  kmsService: Pick<
    TKmsServiceFactory,
    "createCipherPairWithDataKey" | "decryptWithKmsKey" | "generateKmsKey" | "encryptWithKmsKey"
  >;
  appConnectionDAL: Pick<TAppConnectionDALFactory, "findById" | "update" | "updateById">;
  keyStore: Pick<
    TKeyStoreFactory,
    "acquireLock" | "incrementByAndRefreshExpiryIfUnderLimit" | "decrementByOrDelete" | "getItem" | "setItemWithExpiry"
  >;
  pkiSyncDAL: Pick<
    TPkiSyncDALFactory,
    "findById" | "find" | "updateById" | "deleteById" | "update" | "findFailureNotificationRecipients" | "primaryNode"
  >;
  auditLogService: Pick<TAuditLogServiceFactory, "createAuditLog">;
  notificationService: Pick<TNotificationServiceFactory, "createUserNotifications">;
  pkiApplicationDAL: Pick<TPkiApplicationDALFactory, "findById">;
  projectDAL: TProjectDALFactory;
  certificateDAL: TCertificateDALFactory;
  certificateBodyDAL: Pick<TCertificateBodyDALFactory, "findOne" | "create">;
  certificateSecretDAL: Pick<TCertificateSecretDALFactory, "findOne" | "create">;
  certificateAuthorityDAL: Pick<TCertificateAuthorityDALFactory, "findById">;
  certificateAuthorityCertDAL: Pick<TCertificateAuthorityCertDALFactory, "findById">;
  certificateSyncDAL: TCertificateSyncDALFactory;
  gatewayV2Service?: Pick<TGatewayV2ServiceFactory, "getPlatformConnectionDetailsByGatewayId" | "getGatewayById">;
  gatewayPoolService?: Pick<TGatewayPoolServiceFactory, "resolveEffectiveGatewayId">;
  telemetryService: Pick<TTelemetryServiceFactory, "sendPostHogEvents">;
  oidcIssuerService: Pick<TOidcIssuerServiceFactory, "signToken">;
};

type PkiSyncActionJob = Job<
  TQueuePkiSyncSyncCertificatesByIdDTO | TQueuePkiSyncImportCertificatesByIdDTO | TQueuePkiSyncRemoveCertificatesByIdDTO
>;

const JITTER_MS = 10 * 1000;
const HOST_SERIALISATION_LOCK_TTL_MS = 5 * 60 * 1000;
const PKI_SYNC_LOCK_TTL_MS = 5 * 60 * 1000;

const REQUEUE_MS = 30 * 1000;
const REQUEUE_LIMIT = 30;

const getRequeueDelay = (failureCount?: number) => {
  const jitter = Math.random() * JITTER_MS;
  if (!failureCount) return jitter;
  return REQUEUE_MS + jitter;
};

export const pkiSyncQueueFactory = ({
  queueService,
  kmsService,
  appConnectionDAL,
  keyStore,
  pkiSyncDAL,
  auditLogService,
  notificationService,
  pkiApplicationDAL,
  projectDAL,
  certificateDAL,
  certificateBodyDAL,
  certificateSecretDAL,
  certificateAuthorityDAL,
  certificateAuthorityCertDAL,
  certificateSyncDAL,
  gatewayV2Service,
  gatewayPoolService,
  telemetryService,
  oidcIssuerService
}: TPkiSyncQueueFactoryDep) => {
  const appCfg = getConfig();

  const integrationMeter = highCardinalityMeter("PkiSyncs");
  const syncCertificatesErrorHistogram = integrationMeter.createHistogram("pki_sync_sync_certificates_errors", {
    description: "PKI Sync - sync certificates errors",
    unit: "1"
  });
  const importCertificatesErrorHistogram = integrationMeter.createHistogram("pki_sync_import_certificates_errors", {
    description: "PKI Sync - import certificates errors",
    unit: "1"
  });
  const removeCertificatesErrorHistogram = integrationMeter.createHistogram("pki_sync_remove_certificates_errors", {
    description: "PKI Sync - remove certificates errors",
    unit: "1"
  });

  const $tryAdmitConnectionConcurrency = (connectionId: string, targetHost?: string) =>
    tryAdmitAppConnectionConcurrency(keyStore, connectionId, targetHost);

  const $releaseConnectionConcurrency = (connectionId: string, targetHost?: string) =>
    releaseAppConnectionConcurrency(keyStore, connectionId, targetHost);

  const $certificatesForSync = (pkiSync: TPkiSyncRaw) =>
    buildCertificateMap(pkiSync, {
      certificateDAL,
      certificateBodyDAL,
      certificateSecretDAL,
      certificateAuthorityDAL,
      certificateAuthorityCertDAL,
      certificateSyncDAL,
      projectDAL,
      kmsService
    });

  const queuePkiSyncSyncCertificatesById = async (payload: TQueuePkiSyncSyncCertificatesByIdDTO) =>
    queueService.queue(QueueName.PkiSync, QueueJobs.PkiSyncSyncCertificates, payload, {
      delay: getRequeueDelay(payload.failedToAcquireLockCount), // this is for delaying re-queued jobs if sync is locked
      attempts: 5,
      backoff: {
        type: "exponential",
        delay: 3000
      },
      jobId: randomUUID(),
      removeOnComplete: true,
      removeOnFail: true
    });

  const queuePkiSyncLinkMatchingCertificates = async (payload: TQueuePkiSyncLinkMatchingCertificatesDTO) =>
    queueService.queue(QueueName.PkiSync, QueueJobs.PkiSyncLinkMatchingCertificates, payload, {
      attempts: 5,
      backoff: {
        type: "exponential",
        delay: 3000
      },
      jobId: `pki-sync-link-${payload.certificateId}-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`,
      removeOnComplete: true,
      removeOnFail: true
    });

  const queuePkiSyncReconcileFilters = async (payload: TQueuePkiSyncReconcileFiltersDTO) =>
    queueService.queue(QueueName.PkiSync, QueueJobs.PkiSyncReconcileFilters, payload, {
      attempts: 3,
      backoff: {
        type: "exponential",
        delay: 3000
      },
      jobId: `pki-sync-reconcile-filters-${payload.syncId}`,
      removeOnComplete: true,
      removeOnFail: true
    });

  const queuePkiSyncImportCertificatesById = async (payload: TQueuePkiSyncImportCertificatesByIdDTO) =>
    queueService.queue(QueueName.PkiSync, QueueJobs.PkiSyncImportCertificates, payload, {
      attempts: 5,
      backoff: {
        type: "exponential",
        delay: 3000
      },
      jobId: randomUUID(),
      removeOnComplete: true,
      removeOnFail: true
    });

  const queuePkiSyncRemoveCertificatesById = async (payload: TQueuePkiSyncRemoveCertificatesByIdDTO) =>
    queueService.queue(QueueName.PkiSync, QueueJobs.PkiSyncRemoveCertificates, payload, {
      attempts: 5,
      backoff: {
        type: "exponential",
        delay: 3000
      },
      jobId: randomUUID(),
      removeOnComplete: true,
      removeOnFail: true
    });

  const $importCertificates = async (): Promise<TCertificateMap> => {
    throw new Error("Certificate import functionality is not implemented");
  };

  const $handleSyncCertificatesJob = async (job: TPkiSyncSyncCertificatesDTO, pkiSync: TPkiSyncRaw) => {
    const {
      data: { syncId, auditLogInfo }
    } = job;

    await pkiSyncDAL.updateById(syncId, {
      syncStatus: PkiSyncStatus.Running
    });

    logger.info(
      `PkiSync Sync [syncId=${pkiSync.id}] [destination=${pkiSync.destination}] [projectId=${pkiSync.projectId}] [subscriberId=${pkiSync.subscriberId}] [connectionId=${pkiSync.connectionId}]`
    );

    let isSynced = false;
    let certSyncFailureCount = 0;
    let syncMessage: string | null = null;
    let partialFailure = false;
    let postSyncCommandResult: TPostSyncCommandResult | undefined;
    let healthCheckResult: THealthCheckCommandResult | undefined;
    let isFinalAttempt = job.attemptsStarted === job.opts.attempts;

    const configuredPostSyncCommand = getPostSyncCommand(pkiSync.syncOptions);
    const configuredHealthCheckCommand = getHealthCheckCommand(pkiSync.syncOptions);

    try {
      const pkiSyncWithCredentials = await hydratePkiSyncCredentials({
        pkiSync,
        appConnectionDAL,
        projectDAL,
        kmsService
      });

      const { certificateMap, certificateMetadata } = await $certificatesForSync(pkiSync);

      const statusUpdates = Array.from(certificateMetadata.entries()).map(([, metadata]) => ({
        pkiSyncId: pkiSync.id,
        certificateId: metadata.id,
        status: CertificateSyncStatus.Running,
        message: "Syncing certificate to destination"
      }));

      if (statusUpdates.length > 0) {
        await certificateSyncDAL.bulkUpdateSyncStatus(statusUpdates);
      }

      const syncResult = await PkiSyncFns.syncCertificates(pkiSyncWithCredentials, certificateMap, {
        appConnectionDAL,
        kmsService,
        certificateDAL,
        certificateSyncDAL,
        gatewayV2Service,
        gatewayPoolService,
        keyStore,
        oidcIssuerService
      });

      logger.info(
        {
          syncId: pkiSync.id,
          uploaded: syncResult.uploaded || 0,
          removed: syncResult.removed || 0,
          failedRemovals: syncResult.failedRemovals || 0,
          skipped: syncResult.skipped || 0
        },
        "PKI sync operation completed with certificate cleanup"
      );

      const postSyncUpdates: Array<{
        pkiSyncId: string;
        certificateId: string;
        status: string;
        message?: string;
      }> = [];

      for (const [, metadata] of certificateMetadata.entries()) {
        postSyncUpdates.push({
          pkiSyncId: pkiSync.id,
          certificateId: metadata.id,
          status: CertificateSyncStatus.Succeeded,
          message: "Certificate successfully synced to destination"
        });
      }

      if (syncResult.details?.validationErrors) {
        for (const validationError of syncResult.details.validationErrors) {
          const metadata = certificateMetadata.get(validationError.name);
          if (metadata) {
            const updateIndex = postSyncUpdates.findIndex((u) => u.certificateId === metadata.id);
            if (updateIndex >= 0) {
              postSyncUpdates[updateIndex] = {
                pkiSyncId: pkiSync.id,
                certificateId: metadata.id,
                status: CertificateSyncStatus.Failed,
                message: `${validationError.error}`
              };
            }
          }
        }
      }

      if (syncResult.details?.failedUploads) {
        for (const failure of syncResult.details.failedUploads) {
          const metadata = certificateMetadata.get(failure.name);
          if (metadata) {
            const updateIndex = postSyncUpdates.findIndex((u) => u.certificateId === metadata.id);
            if (updateIndex >= 0) {
              postSyncUpdates[updateIndex] = {
                pkiSyncId: pkiSync.id,
                certificateId: metadata.id,
                status: CertificateSyncStatus.Failed,
                message: `Failed to sync certificate: ${failure.error}`
              };
            }
          }
        }
      }

      if (syncResult.details?.skippedCertificates) {
        for (const skip of syncResult.details.skippedCertificates) {
          const metadata = certificateMetadata.get(skip.name);
          if (metadata) {
            const updateIndex = postSyncUpdates.findIndex((u) => u.certificateId === metadata.id);
            if (updateIndex >= 0) {
              postSyncUpdates[updateIndex] = {
                pkiSyncId: pkiSync.id,
                certificateId: metadata.id,
                status: CertificateSyncStatus.Failed,
                message: `Certificate skipped: ${skip.reason}`
              };
            }
          }
        }
      }

      if (postSyncUpdates.length > 0) {
        await certificateSyncDAL.bulkUpdateSyncStatus(postSyncUpdates);
      }

      const failedCertificateCount = postSyncUpdates.filter(
        (update) => update.status === CertificateSyncStatus.Failed
      ).length;
      certSyncFailureCount = failedCertificateCount + (syncResult.failedRemovals ?? 0);

      const processedCertificateIds = new Set(Array.from(certificateMetadata.values()).map((meta) => meta.id));
      const nonTerminalStatuses = new Set<string>([
        CertificateSyncStatus.Pending,
        CertificateSyncStatus.Running,
        CertificateSyncStatus.Syncing
      ]);
      const trackedRecords = await certificateSyncDAL.findByPkiSyncId(pkiSync.id);
      const strandedCertificateIds = trackedRecords
        .filter(
          (record) =>
            record.certificateId &&
            !processedCertificateIds.has(record.certificateId) &&
            nonTerminalStatuses.has(record.syncStatus ?? "")
        )
        .map((record) => record.certificateId)
        .filter((id): id is string => typeof id === "string");
      if (strandedCertificateIds.length > 0) {
        const stillEligible = await certificateDAL.findActiveCertificatesByIds(strandedCertificateIds);
        if (stillEligible.length > 0) {
          await certificateSyncDAL.bulkUpdateSyncStatus(
            stillEligible.map((cert) => ({
              pkiSyncId: pkiSync.id,
              certificateId: cert.id,
              status: CertificateSyncStatus.Failed,
              message:
                "Certificate could not be prepared for syncing (its data could not be loaded, or it resolved to the same file name as another certificate)"
            }))
          );
          certSyncFailureCount += stillEligible.length;
        }
      }

      postSyncCommandResult = syncResult.postSyncCommand;
      partialFailure = Boolean(syncResult.partialFailureMessage);
      healthCheckResult = syncResult.healthCheck;

      const reasons =
        healthCheckResult && didHealthCheckFail(healthCheckResult)
          ? [buildHealthCheckCommandFailureMessage(healthCheckResult)]
          : [
              certSyncFailureCount > 0
                ? `${certSyncFailureCount} certificate(s) failed to sync to the destination`
                : null,
              syncResult.partialFailureMessage ?? null,
              postSyncCommandResult?.status === PkiSyncStatus.Failed
                ? buildPostSyncCommandFailureMessage(postSyncCommandResult)
                : null
            ].filter(Boolean);

      if (reasons.length > 0) {
        syncMessage = truncateSyncMessage(reasons.join(". "));
      }

      isSynced = true;
    } catch (err) {
      logger.error(
        err,
        `PkiSync Sync Error [syncId=${pkiSync.id}] [destination=${pkiSync.destination}] [projectId=${pkiSync.projectId}] [subscriberId=${pkiSync.subscriberId}] [connectionId=${pkiSync.connectionId}]`
      );

      if (appCfg.OTEL_TELEMETRY_COLLECTION_ENABLED) {
        syncCertificatesErrorHistogram.record(1, {
          version: 1,
          destination: pkiSync.destination,
          syncId: pkiSync.id,
          projectId: pkiSync.projectId,
          type: err instanceof AxiosError ? "AxiosError" : err?.constructor?.name || "UnknownError",
          status: err instanceof AxiosError ? err.response?.status : undefined,
          name: err instanceof Error ? err.name : undefined
        });
      }

      syncMessage = parsePkiSyncErrorMessage(err);

      if (err instanceof PkiSyncError && !err.shouldRetry) {
        isFinalAttempt = true;
      } else {
        throw err;
      }
    } finally {
      const ranAt = new Date();
      const postSyncCommandFailed = postSyncCommandResult?.status === PkiSyncStatus.Failed;
      const fullySynced =
        isSynced &&
        certSyncFailureCount === 0 &&
        !postSyncCommandFailed &&
        !partialFailure &&
        !didHealthCheckFail(healthCheckResult);
      const syncStatus = fullySynced ? PkiSyncStatus.Succeeded : PkiSyncStatus.Failed;

      await auditLogService.createAuditLog({
        projectId: pkiSync.projectId,
        ...(auditLogInfo ?? {
          actor: {
            type: ActorType.PLATFORM,
            metadata: {}
          }
        }),
        event: {
          type: EventType.PKI_SYNC_SYNC_CERTIFICATES,
          metadata: {
            syncId: pkiSync.id,
            syncMessage,
            jobId: job.id!,
            jobRanAt: ranAt,
            healthCheck: configuredHealthCheckCommand
              ? { command: configuredHealthCheckCommand, result: healthCheckResult }
              : undefined,
            postSyncCommand: configuredPostSyncCommand
              ? { command: configuredPostSyncCommand, result: postSyncCommandResult }
              : undefined
          }
        }
      });

      if (isSynced || isFinalAttempt) {
        if (!fullySynced) {
          let failureKind = PkiSyncFailureKind.Sync;
          if (didHealthCheckFail(healthCheckResult)) failureKind = PkiSyncFailureKind.HealthCheck;
          else if (postSyncCommandFailed) failureKind = PkiSyncFailureKind.PostSyncCommand;

          await notifyPkiSyncFailure(
            { pkiSync, kind: failureKind, message: syncMessage ?? "The sync did not complete." },
            { pkiSyncDAL, projectDAL, pkiApplicationDAL, notificationService }
          );
        }

        await pkiSyncDAL.updateById(pkiSync.id, {
          syncStatus,
          lastSyncJobId: job.id,
          lastSyncMessage: syncMessage,
          lastSyncedAt: fullySynced ? ranAt : undefined,
          ...(healthCheckResult
            ? {
                lastHealthCheckRanAt: ranAt,
                lastHealthCheckStatus: didHealthCheckFail(healthCheckResult)
                  ? PkiSyncStatus.Failed
                  : PkiSyncStatus.Succeeded,
                lastHealthCheckMessage: didHealthCheckFail(healthCheckResult)
                  ? truncateSyncMessage(buildHealthCheckCommandFailureMessage(healthCheckResult))
                  : null
              }
            : {})
        });

        await telemetryService.sendPostHogEvents({
          event: PostHogEventTypes.PkiSyncExecuted,
          distinctId: `platform/${pkiSync.projectId}`,
          organizationId: pkiSync.connection.orgId,
          properties: {
            orgId: pkiSync.connection.orgId,
            projectId: pkiSync.projectId,
            destination: pkiSync.destination,
            success: fullySynced
          }
        });
      }
    }
  };

  const $handleImportCertificatesJob = async (job: TPkiSyncImportCertificatesDTO, pkiSync: TPkiSyncRaw) => {
    const {
      data: { syncId, auditLogInfo }
    } = job;

    await pkiSyncDAL.updateById(syncId, {
      importStatus: PkiSyncStatus.Running
    });

    logger.info(
      `PkiSync Import [syncId=${pkiSync.id}] [destination=${pkiSync.destination}] [projectId=${pkiSync.projectId}] [subscriberId=${pkiSync.subscriberId}] [connectionId=${pkiSync.connectionId}]`
    );

    let isSuccess = false;
    let importMessage: string | null = null;
    let isFinalAttempt = job.attemptsStarted === job.opts.attempts;

    try {
      await $importCertificates();

      isSuccess = true;
    } catch (err) {
      logger.error(
        err,
        `PkiSync Import Error [syncId=${pkiSync.id}] [destination=${pkiSync.destination}] [projectId=${pkiSync.projectId}] [subscriberId=${pkiSync.subscriberId}] [connectionId=${pkiSync.connectionId}]`
      );

      if (appCfg.OTEL_TELEMETRY_COLLECTION_ENABLED) {
        importCertificatesErrorHistogram.record(1, {
          version: 1,
          destination: pkiSync.destination,
          syncId: pkiSync.id,
          projectId: pkiSync.projectId,
          type: err instanceof AxiosError ? "AxiosError" : err?.constructor?.name || "UnknownError",
          status: err instanceof AxiosError ? err.response?.status : undefined,
          name: err instanceof Error ? err.name : undefined
        });
      }

      importMessage = parsePkiSyncErrorMessage(err);

      if (err instanceof PkiSyncError && !err.shouldRetry) {
        isFinalAttempt = true;
      } else {
        throw err;
      }
    } finally {
      const ranAt = new Date();
      const importStatus = isSuccess ? PkiSyncStatus.Succeeded : PkiSyncStatus.Failed;

      await auditLogService.createAuditLog({
        projectId: pkiSync.projectId,
        ...(auditLogInfo ?? {
          actor: {
            type: ActorType.PLATFORM,
            metadata: {}
          }
        }),
        event: {
          type: EventType.PKI_SYNC_IMPORT_CERTIFICATES,
          metadata: {
            syncId: pkiSync.id,
            importMessage,
            jobId: job.id!,
            jobRanAt: ranAt
          }
        }
      });

      if (isSuccess || isFinalAttempt) {
        await pkiSyncDAL.updateById(pkiSync.id, {
          importStatus,
          lastImportJobId: job.id,
          lastImportMessage: importMessage,
          lastImportedAt: isSuccess ? ranAt : undefined
        });
      }
    }
  };

  const $handleRemoveCertificatesJob = async (job: TPkiSyncRemoveCertificatesDTO, pkiSync: TPkiSyncRaw) => {
    const {
      data: { syncId, auditLogInfo, deleteSyncOnComplete, certificateIds: certificateIdsToRemove, removalOptions }
    } = job;

    await pkiSyncDAL.updateById(syncId, {
      removeStatus: PkiSyncStatus.Running
    });

    logger.info(
      `PkiSync Remove [syncId=${pkiSync.id}] [destination=${pkiSync.destination}] [projectId=${pkiSync.projectId}] [subscriberId=${pkiSync.subscriberId}] [connectionId=${pkiSync.connectionId}]`
    );

    let isSuccess = false;
    let removeMessage: string | null = null;
    let isFinalAttempt = job.attemptsStarted === job.opts.attempts;

    try {
      const pkiSyncWithCredentials = await hydratePkiSyncCredentials({
        pkiSync,
        appConnectionDAL,
        projectDAL,
        kmsService
      });

      const certificateMap: TCertificateMap = certificateIdsToRemove?.length
        ? Object.fromEntries(
            certificateIdsToRemove.map((certId, index) => [
              `certificate-${index}`,
              { cert: "", privateKey: "", certificateId: certId }
            ])
          )
        : (await $certificatesForSync(pkiSync)).certificateMap;

      await PkiSyncFns.removeCertificates(pkiSyncWithCredentials, Object.keys(certificateMap), {
        appConnectionDAL,
        kmsService,
        certificateSyncDAL,
        certificateDAL,
        certificateMap,
        removalOptions,
        gatewayV2Service,
        gatewayPoolService,
        keyStore,
        oidcIssuerService
      });

      isSuccess = true;
    } catch (err) {
      logger.error(
        err,
        `PkiSync Remove Error [syncId=${pkiSync.id}] [destination=${pkiSync.destination}] [projectId=${pkiSync.projectId}] [subscriberId=${pkiSync.subscriberId}] [connectionId=${pkiSync.connectionId}]`
      );

      if (appCfg.OTEL_TELEMETRY_COLLECTION_ENABLED) {
        removeCertificatesErrorHistogram.record(1, {
          version: 1,
          destination: pkiSync.destination,
          syncId: pkiSync.id,
          projectId: pkiSync.projectId,
          type: err instanceof AxiosError ? "AxiosError" : err?.constructor?.name || "UnknownError",
          status: err instanceof AxiosError ? err.response?.status : undefined,
          name: err instanceof Error ? err.name : undefined
        });
      }

      removeMessage = parsePkiSyncErrorMessage(err);

      if (err instanceof PkiSyncError && !err.shouldRetry) {
        isFinalAttempt = true;
      } else {
        throw err;
      }
    } finally {
      const ranAt = new Date();
      const removeStatus = isSuccess ? PkiSyncStatus.Succeeded : PkiSyncStatus.Failed;

      await auditLogService.createAuditLog({
        projectId: pkiSync.projectId,
        ...(auditLogInfo ?? {
          actor: {
            type: ActorType.PLATFORM,
            metadata: {}
          }
        }),
        event: {
          type: EventType.PKI_SYNC_REMOVE_CERTIFICATES,
          metadata: {
            syncId: pkiSync.id,
            removeMessage,
            jobId: job.id!,
            jobRanAt: ranAt
          }
        }
      });

      if (isSuccess || isFinalAttempt) {
        if (isSuccess && deleteSyncOnComplete) {
          await pkiSyncDAL.deleteById(pkiSync.id);
        } else {
          await pkiSyncDAL.updateById(pkiSync.id, {
            removeStatus,
            lastRemoveJobId: job.id,
            lastRemoveMessage: removeMessage,
            lastRemovedAt: isSuccess ? ranAt : undefined
          });
        }
      }
    }
  };

  const $keepLocksAlive = (syncId: string, locks: Array<TKeyStoreLock | null>) => {
    let held = locks.filter((lock): lock is TKeyStoreLock => Boolean(lock));
    const timer = setInterval(() => {
      void Promise.all(
        held.map((current) =>
          current.extend
            ? current.extend(PKI_SYNC_LOCK_TTL_MS).catch((error: unknown) => {
                logger.warn(error, `Failed to extend PKI sync lock [syncId=${syncId}]`);
                return current;
              })
            : Promise.resolve(current)
        )
      ).then((extended) => {
        held = extended;
      });
    }, PKI_SYNC_LOCK_TTL_MS / 4);
    return { stop: () => clearInterval(timer) };
  };

  const $handleAcquireLockFailure = async (job: PkiSyncActionJob) => {
    const { syncId } = job.data;

    switch (job.name) {
      case QueueJobs.PkiSyncSyncCertificates: {
        const { failedToAcquireLockCount = 0, ...rest } = job.data as TQueuePkiSyncSyncCertificatesByIdDTO;

        if (failedToAcquireLockCount < REQUEUE_LIMIT) {
          const current = await pkiSyncDAL.findById(syncId);
          if (current?.syncStatus !== PkiSyncStatus.Running) {
            await pkiSyncDAL.updateById(syncId, { syncStatus: PkiSyncStatus.Pending, lastSyncMessage: null });
          }
          await queuePkiSyncSyncCertificatesById({ ...rest, failedToAcquireLockCount: failedToAcquireLockCount + 1 });
          return;
        }

        await pkiSyncDAL.updateById(syncId, {
          syncStatus: PkiSyncStatus.Failed,
          lastSyncMessage:
            "Failed to run job. This typically happens when a sync is already in progress. Please try again.",
          lastSyncJobId: job.id
        });

        break;
      }
      case QueueJobs.PkiSyncImportCertificates: {
        await pkiSyncDAL.updateById(syncId, {
          importStatus: PkiSyncStatus.Failed,
          lastImportMessage:
            "Failed to run job. This typically happens when a sync is already in progress. Please try again.",
          lastImportJobId: job.id
        });

        break;
      }
      case QueueJobs.PkiSyncRemoveCertificates: {
        const { failedToAcquireLockCount = 0, ...rest } = job.data as TQueuePkiSyncRemoveCertificatesByIdDTO;

        if (failedToAcquireLockCount < REQUEUE_LIMIT) {
          await queuePkiSyncRemoveCertificatesById({
            ...rest,
            failedToAcquireLockCount: failedToAcquireLockCount + 1
          });
          return;
        }

        await pkiSyncDAL.updateById(syncId, {
          removeStatus: PkiSyncStatus.Failed,
          lastRemoveMessage:
            "Failed to run job. This typically happens when a sync is already in progress. Please try again.",
          lastRemoveJobId: job.id
        });

        break;
      }
      default:
        throw new Error(`Unhandled PKI Sync Job ${String(job.name)}`);
    }
  };

  queueService.start(QueueName.PkiSync, async (job) => {
    if (job.name === QueueJobs.PkiSyncLinkMatchingCertificates) {
      const { certificateId, applicationId } = job.data as TQueuePkiSyncLinkMatchingCertificatesDTO;
      try {
        await reconcileCertificateAgainstMatchingSyncs(certificateId, applicationId, {
          certificateDAL,
          certificateSyncDAL,
          pkiSyncDAL,
          pkiSyncQueue: { queuePkiSyncSyncCertificatesById, queuePkiSyncRemoveCertificatesById },
          auditLogService,
          pkiApplicationDAL,
          withSyncFilterLock: (syncId, run) => withPkiSyncFilterLock(keyStore, syncId, run)
        });
      } catch (error) {
        logger.error(
          error,
          `Failed to reconcile certificate against matching PKI syncs [certificateId=${certificateId}] [applicationId=${applicationId}]`
        );
        throw error;
      }
      return;
    }

    if (job.name === QueueJobs.PkiSyncReconcileFilters) {
      const { syncId: reconcileSyncId } = job.data as TQueuePkiSyncReconcileFiltersDTO;
      await reconcileSyncFilters(reconcileSyncId, {
        certificateDAL,
        certificateSyncDAL,
        pkiSyncDAL,
        pkiSyncQueue: { queuePkiSyncSyncCertificatesById, queuePkiSyncRemoveCertificatesById },
        auditLogService,
        pkiApplicationDAL,
        withSyncFilterLock: (lockSyncId, run) => withPkiSyncFilterLock(keyStore, lockSyncId, run)
      });
      return;
    }

    const { syncId } = job.data as PkiSyncActionJob["data"];

    const pkiSync = await pkiSyncDAL.findById(syncId);

    if (!pkiSync) throw new Error(`Cannot find PKI sync with ID ${syncId}`);

    const { connectionId } = pkiSync;

    const needsConnectionSlot = job.name === QueueJobs.PkiSyncSyncCertificates;

    const needsHostSerialisation =
      (needsConnectionSlot && getPkiSyncProviderCapabilities(pkiSync.destination).canRunHealthCheckCommand) ||
      isPkiSyncSerialisedPerConnection(pkiSync.destination);

    const targetHost = getPkiSyncTargetHost(pkiSync.destinationConfig);

    let connectionLock: Awaited<ReturnType<typeof keyStore.acquireLock>> | null = null;
    if (needsHostSerialisation) {
      connectionLock = await keyStore
        .acquireLock(
          [KeyStorePrefixes.AppConnectionCommandLock(connectionId, targetHost)],
          HOST_SERIALISATION_LOCK_TTL_MS,
          PKI_SYNC_CONNECTION_LOCK_RETRY
        )
        .catch(() => null);

      if (!connectionLock) {
        await $handleAcquireLockFailure(job as PkiSyncActionJob);

        return;
      }
    }

    if (needsConnectionSlot && !(await $tryAdmitConnectionConcurrency(connectionId, targetHost))) {
      await connectionLock?.release();
      await $handleAcquireLockFailure(job as PkiSyncActionJob);

      return;
    }

    let lock: Awaited<ReturnType<typeof keyStore.acquireLock>>;

    try {
      lock = await keyStore.acquireLock(
        [KeyStorePrefixes.PkiSyncLock(syncId)],
        // PKI syncs can take excessive amounts of time so we need to keep it locked
        PKI_SYNC_LOCK_TTL_MS
      );
    } catch (e) {
      if (needsConnectionSlot) await $releaseConnectionConcurrency(connectionId, targetHost);
      await connectionLock?.release();
      await $handleAcquireLockFailure(job as PkiSyncActionJob);

      return;
    }

    const keepAlive = isPkiSyncSerialisedPerConnection(pkiSync.destination)
      ? $keepLocksAlive(syncId, [lock, connectionLock])
      : undefined;

    try {
      switch (job.name) {
        case QueueJobs.PkiSyncSyncCertificates: {
          await $handleSyncCertificatesJob(job as TPkiSyncSyncCertificatesDTO, pkiSync);
          break;
        }
        case QueueJobs.PkiSyncImportCertificates:
          await $handleImportCertificatesJob(job as TPkiSyncImportCertificatesDTO, pkiSync);
          break;
        case QueueJobs.PkiSyncRemoveCertificates:
          await $handleRemoveCertificatesJob(job as TPkiSyncRemoveCertificatesDTO, pkiSync);
          break;
        default:
          throw new Error(`Unhandled PKI Sync Job ${String(job.name)}`);
      }
    } finally {
      keepAlive?.stop();
      if (needsConnectionSlot) await $releaseConnectionConcurrency(connectionId, targetHost);

      await Promise.allSettled([lock.release(), connectionLock?.release()]);
    }
  });

  return {
    queuePkiSyncLinkMatchingCertificates,
    queuePkiSyncReconcileFilters,
    queuePkiSyncSyncCertificatesById,
    queuePkiSyncImportCertificatesById,
    queuePkiSyncRemoveCertificatesById
  };
};