Infisical is the open-source platform for securing secrets, certificates, access, and AI Agents.
TypeScript acmeagentsai-agentscertificate-managementdevsecopsenvironment-variableskubernetesopen-sourcepampkipostgresprivate-caprivileged-access-managementsecret-managementsecret-managersecretssecrets-managementsecuritysecurity-toolsvault
import {
Job,
JobSchedulerJson,
MetricsTime,
Queue,
QueueOptions,
RepeatOptions,
Worker,
WorkerListener,
WorkerOptions
} from "bullmq";
import { SecretEncryptionAlgo, SecretKeyEncoding } from "@app/db/schemas";
import { TAuditLogStreamFlushJobData } from "@app/ee/services/audit-log-stream-outbox/audit-log-stream-outbox-types";
import { TAuditLogCollapsedFlushJobData } from "@app/ee/services/audit-log/audit-log-types";
import {
TSecretRotationRotateSecretsJobPayload,
TSecretRotationSendNotificationJobPayload
} from "@app/ee/services/secret-rotation-v2/secret-rotation-v2-types";
import {
TQueueSecretScanningDataSourceEvent,
TQueueSecretScanningDataSourceFullScan,
TQueueSecretScanningResourceDiffScan,
TQueueSecretScanningResourceDiffScanPayload,
TQueueSecretScanningSendNotification
} from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-types";
import {
TScanFullRepoEventPayload,
TScanPushEventPayload
} from "@app/ee/services/secret-scanning/secret-scanning-queue/secret-scanning-queue-types";
import { getConfig } from "@app/lib/config/env";
import { buildRedisFromConfig, createThrottledRedisErrorLogger, TRedisConfigKeys } from "@app/lib/config/redis";
import { crypto } from "@app/lib/crypto";
import { classifyError } from "@app/lib/errors/classify";
import { logger } from "@app/lib/logger";
import {
queueJobCounter,
queueJobDurationHistogram,
queueJobFailureCounter,
queueJobWaitHistogram,
queueStalledCounter,
resolveCoreMeter
} from "@app/lib/telemetry/metrics";
import {
TAppConnectionCredentialRotationRotateJobPayload,
TAppConnectionCredentialRotationSendNotificationJobPayload
} from "@app/services/app-connection/credential-rotation/app-connection-credential-rotation-types";
import { CaType } from "@app/services/certificate-authority/certificate-authority-enums";
import {
ExternalPlatforms,
TVaultImportSideEffectsJobPayload
} from "@app/services/external-migration/external-migration-types";
import { TCreateUserNotificationDTO } from "@app/services/notification/notification-types";
import { PkiAlertEventType } from "@app/services/pki-alert-v2/pki-alert-v2-types";
import {
TQueuePkiSyncImportCertificatesByIdDTO,
TQueuePkiSyncLinkMatchingCertificatesDTO,
TQueuePkiSyncReconcileFiltersDTO,
TQueuePkiSyncRemoveCertificatesByIdDTO,
TQueuePkiSyncRunHealthCheckByIdDTO,
TQueuePkiSyncSyncCertificatesByIdDTO
} from "@app/services/pki-sync/pki-sync-types";
import {
TQueueSecretSyncImportSecretsByIdDTO,
TQueueSecretSyncRemoveSecretsByIdDTO,
TQueueSecretSyncSyncSecretsByIdDTO,
TQueueSendSecretSyncActionFailedNotificationsDTO
} from "@app/services/secret-sync/secret-sync-types";
import {
TFailedIntegrationSyncEmailsPayload,
TIntegrationSyncPayload,
TSyncSecretsDTO
} from "@app/services/secret/secret-types";
import { CacheType } from "@app/services/super-admin/super-admin-types";
import { TWebhookPayloads } from "@app/services/webhook/webhook-types";
// Scheduler IDs are prefixed so they never collide with legacy repeatable-job
// iteration IDs that may still be pending in Redis after the migration.
// Bump the version if a future migration needs the same trick.
export const JOB_SCHEDULER_PREFIX = "jsv1";
export enum QueueName {
SecretReminder = "secret-reminder",
// TODO(akhilmhdh): This will get removed later. For now this is kept to stop the repeatable queue
AuditLogPrune = "audit-log-prune",
PkiAlertV2Event = "pki-alert-v2-event",
PkiSubscriber = "pki-subscriber",
IntegrationSync = "sync-integrations",
SecretWebhook = "secret-webhook",
SecretFullRepoScan = "secret-full-repo-scan",
SecretPushEventScan = "secret-push-event-scan",
UpgradeProjectToGhost = "upgrade-project-to-ghost",
DynamicSecretRevocation = "dynamic-secret-revocation",
DynamicSecretLeaseRevocationFailedEmail = "dynamic-secret-lease-revocation-failed-email",
CaLifecycle = "ca-lifecycle", // parent queue to ca-order-certificate-for-subscriber
CertificateIssuance = "certificate-issuance",
SecretReplication = "secret-replication",
SecretSync = "secret-sync", // parent queue to push integration sync, webhook, and secret replication
PkiSync = "pki-sync",
PkiSyncHealthCheck = "pki-sync-health-check",
ProjectV3Migration = "project-v3-migration",
AccessTokenStatusUpdate = "access-token-status-update",
ImportSecretsFromExternalSource = "import-secrets-from-external-source",
VaultImportSideEffects = "vault-import-side-effects",
AppConnectionSecretSync = "app-connection-secret-sync",
SecretRotationV2 = "secret-rotation-v2",
SecretRotationV2RotateSecrets = "secret-rotation-v2-rotate-secrets",
PamCredentialRotation = "pam-credential-rotation",
PamCredentialRotationRotate = "pam-credential-rotation-rotate",
PamHeartbeat = "pam-heartbeat",
PamHeartbeatCheck = "pam-heartbeat-check",
FolderTreeCheckpoint = "folder-tree-checkpoint",
InvalidateCache = "invalidate-cache",
SecretScanningV2 = "secret-scanning-v2",
SecretScanningV2FullScan = "secret-scanning-v2-full-scan",
SecretScanningV2RealtimeScan = "secret-scanning-v2-realtime-scan",
UserNotification = "user-notification",
AlertDispatch = "alert-dispatch",
EventOutboxFlush = "event-outbox-flush",
AuditReportGeneration = "audit-report-generation",
PamSessionExpiration = "pam-session-expiration",
PamDiscoveryScan = "pam-discovery-scan",
PkiAcmeChallengeValidation = "pki-acme-challenge-validation",
PkiDiscoveryScan = "pki-discovery-scan",
PkiDiscoveryRescan = "pki-discovery-rescan",
AppConnectionCredentialRotation = "app-connection-credential-rotation",
AppConnectionCredentialRotationRotate = "app-connection-credential-rotation-rotate",
AuditLogClickHouseBatch = "audit-log-clickhouse-batch",
AuditLogStreamOutbox = "audit-log-stream-outbox",
AuditLogCollapsedFlush = "audit-log-collapsed-flush",
CaAutoRenewal = "ca-auto-renewal",
ProjectHardDelete = "project-hard-delete",
ProjectEnvHardDelete = "project-env-hard-delete",
SignerAutoRenewal = "signer-auto-renewal",
SecretBlindIndexMigration = "secret-blind-index-migration",
UsageEvent = "usage-event",
IntegrationDeprecationNotice = "integration-deprecation-notice",
LegacyPkiDeprecationNotice = "legacy-pki-deprecation-notice"
}
export enum QueueJobs {
SecretReminder = "secret-reminder-job",
// TODO(akhilmhdh): This will get removed later. For now this is kept to stop the repeatable queue
AuditLogPrune = "audit-log-prune-job",
DailyResourceCleanUp = "daily-resource-cleanup-job",
FrequentResourceCleanUp = "frequent-resource-cleanup-job",
DailyExpiringPkiItemAlert = "daily-expiring-pki-item-alert",
DailyPkiAlertV2Processing = "daily-pki-alert-v2-processing",
PkiAlertV2ProcessEvent = "pki-alert-v2-process-event",
PkiSyncCleanup = "pki-sync-cleanup-job",
SecWebhook = "secret-webhook-trigger",
TelemetryInstanceStats = "telemetry-self-hosted-stats",
IntegrationSync = "secret-integration-pull",
SendFailedIntegrationSyncEmails = "send-failed-integration-sync-emails",
SecretScan = "secret-scan",
UpgradeProjectToGhost = "upgrade-project-to-ghost-job",
DynamicSecretRevocation = "dynamic-secret-revocation",
DynamicSecretPruning = "dynamic-secret-pruning",
CaCrlRotation = "ca-crl-rotation-job",
SecretReplication = "secret-replication",
SecretSync = "secret-sync", // parent queue to push integration sync, webhook, and secret replication
PkiSync = "pki-sync",
ProjectV3Migration = "project-v3-migration",
IdentityAccessTokenStatusUpdate = "identity-access-token-status-update",
ServiceTokenStatusUpdate = "service-token-status-update",
ImportSecretsFromExternalSource = "import-secrets-from-external-source",
VaultImportSideEffects = "vault-import-side-effects",
SecretSyncSyncSecrets = "secret-sync-sync-secrets",
SecretSyncImportSecrets = "secret-sync-import-secrets",
SecretSyncRemoveSecrets = "secret-sync-remove-secrets",
SecretSyncSendActionFailedNotifications = "secret-sync-send-action-failed-notifications",
PkiSyncSyncCertificates = "pki-sync-sync-certificates",
PkiSyncLinkMatchingCertificates = "pki-sync-link-matching-certificates",
PkiSyncReconcileFilters = "pki-sync-reconcile-filters",
PkiSyncImportCertificates = "pki-sync-import-certificates",
PkiSyncRemoveCertificates = "pki-sync-remove-certificates",
PkiSyncRunHealthCheck = "pki-sync-run-health-check",
SecretRotationV2QueueRotations = "secret-rotation-v2-queue-rotations",
SecretRotationV2RotateSecrets = "secret-rotation-v2-rotate-secrets",
SecretRotationV2SendNotification = "secret-rotation-v2-send-notification",
PamCredentialRotationQueueRotations = "pam-credential-rotation-queue-rotations",
PamCredentialRotationRotate = "pam-credential-rotation-rotate",
PamHeartbeatQueueChecks = "pam-heartbeat-queue-checks",
PamHeartbeatCheck = "pam-heartbeat-check",
CreateFolderTreeCheckpoint = "create-folder-tree-checkpoint",
DynamicSecretLeaseRevocationFailedEmail = "dynamic-secret-lease-revocation-failed-email",
InvalidateCache = "invalidate-cache",
SecretScanningV2FullScan = "secret-scanning-v2-full-scan",
// Kept as "diff-scan" while the queue moved to "realtime-scan": jobs already enqueued under this
// name carry it, and the drain branch on QueueName.SecretScanningV2 matches on it.
SecretScanningV2DiffScan = "secret-scanning-v2-diff-scan",
SecretScanningV2SendNotification = "secret-scanning-v2-notification",
SecretScanningV2DataSourceEvent = "secret-scanning-v2-data-source-event",
CaOrderCertificateForSubscriber = "ca-order-certificate-for-subscriber",
CaIssueCertificateFromProfile = "ca-issue-certificate-from-profile",
PkiSubscriberDailyAutoRenewal = "pki-subscriber-daily-auto-renewal",
TelemetryAggregatedEvents = "telemetry-aggregated-events",
DailyReminders = "daily-reminders",
SecretReminderMigration = "secret-reminder-migration",
UserNotification = "user-notification-job",
AlertDispatch = "alert-dispatch-job",
EventOutboxFlush = "event-outbox-flush-job",
GenerateAuditReport = "generate-audit-report-job",
HealthAlert = "health-alert",
CertificateV3DailyAutoRenewal = "certificate-v3-daily-auto-renewal",
PamSessionExpiration = "pam-session-expiration",
PamDiscoverySourceScan = "pam-discovery-source-scan",
PkiAcmeChallengeValidation = "pki-acme-challenge-validation",
PkiDiscoveryRunScan = "pki-discovery-run-scan",
PkiDiscoveryScheduledScan = "pki-discovery-scheduled-scan",
PkiDiscoveryRescanInstallation = "pki-discovery-rescan-installation",
AppConnectionCredentialRotationQueueRotations = "app-connection-credential-rotation-queue-rotations",
AppConnectionCredentialRotationRotate = "app-connection-credential-rotation-rotate",
AppConnectionCredentialRotationSendNotification = "app-connection-credential-rotation-send-notification",
AuditLogClickHouseBatch = "audit-log-clickhouse-batch-job",
AuditLogStreamFlush = "audit-log-stream-flush",
AuditLogCollapsedFlush = "audit-log-collapsed-flush-job",
CaDailyAutoRenewal = "ca-daily-auto-renewal",
CaVenafiInstall = "ca-venafi-install-job",
CaAdcsInstall = "ca-adcs-install-job",
CaNativeAdcsInstall = "ca-native-adcs-install-job",
CertificateCleanup = "certificate-cleanup-job",
DailySecretSyncRetry = "daily-secret-sync-retry-job",
DigiCertOrderPolling = "digicert-order-polling-job",
GoDaddyOrderPolling = "godaddy-order-polling-job",
ProjectHardDelete = "project-hard-delete-job",
ProjectEnvHardDelete = "project-env-hard-delete-job",
SignerDailyAutoRenewal = "signer-daily-auto-renewal",
SecretBlindIndexMigration = "secret-blind-index-migration",
UsageEvent = "usage-event-job",
SendIntegrationDeprecationNotice = "send-integration-deprecation-notice",
SendLegacyPkiDeprecationNotice = "send-legacy-pki-deprecation-notice"
}
export enum JobState {
NotFound = "not-found",
Pending = "pending",
Completed = "completed",
Failed = "failed"
}
type BaseQueueOptions = {
removeOnComplete?: boolean | { count: number } | { age: number };
removeOnFail?: boolean | { count: number } | { age: number };
attempts?: number;
delay?: number;
backoff?: {
type: "exponential" | "fixed";
delay: number;
};
// @deprecated Use upsertJobScheduler instead.
repeat?: {
pattern?: string;
every?: number;
// only works with every by bullmq
immediately?: boolean;
key: string;
utc?: boolean;
};
};
type DeduplicationOptions = {
id: string;
keepLastIfActive?: boolean;
replace?: boolean;
extend?: boolean;
ttl?: number;
};
export type TQueueOptions = BaseQueueOptions &
(
| {
jobId: string;
deduplication?: undefined;
}
| {
jobId?: undefined;
deduplication: DeduplicationOptions;
}
);
export type TQueueJobTypes = {
[QueueName.SecretReminder]: {
payload: {
projectId: string;
secretId: string;
repeatDays: number;
note: string | undefined | null;
};
name: QueueJobs.SecretReminder;
};
[QueueName.PkiAlertV2Event]: {
name: QueueJobs.PkiAlertV2ProcessEvent;
payload: {
certificateId: string;
projectId: string;
eventType: PkiAlertEventType;
};
};
[QueueName.AuditLogPrune]: {
name: QueueJobs.AuditLogPrune;
payload: undefined;
};
[QueueName.SecretWebhook]: {
name: QueueJobs.SecWebhook;
payload: TWebhookPayloads;
};
[QueueName.AccessTokenStatusUpdate]:
| {
name: QueueJobs.IdentityAccessTokenStatusUpdate;
payload: { identityAccessTokenId: string; numberOfUses: number };
}
| {
name: QueueJobs.ServiceTokenStatusUpdate;
payload: { serviceTokenId: string };
};
[QueueName.IntegrationSync]:
| {
name: QueueJobs.IntegrationSync;
payload: TIntegrationSyncPayload;
}
| {
name: QueueJobs.SendFailedIntegrationSyncEmails;
payload: TFailedIntegrationSyncEmailsPayload;
};
[QueueName.SecretFullRepoScan]: {
name: QueueJobs.SecretScan;
payload: TScanFullRepoEventPayload;
};
[QueueName.SecretPushEventScan]: { name: QueueJobs.SecretScan; payload: TScanPushEventPayload };
[QueueName.UpgradeProjectToGhost]: {
name: QueueJobs.UpgradeProjectToGhost;
payload: {
projectId: string;
startedByUserId: string;
encryptedPrivateKey: {
encryptedKey: string;
encryptedKeyIv: string;
encryptedKeyTag: string;
keyEncoding: SecretKeyEncoding;
};
};
};
[QueueName.DynamicSecretLeaseRevocationFailedEmail]: {
name: QueueJobs.DynamicSecretLeaseRevocationFailedEmail;
payload: {
leaseId: string;
};
};
[QueueName.DynamicSecretRevocation]:
| {
name: QueueJobs.DynamicSecretRevocation;
payload: {
isRetry?: boolean;
leaseId: string;
dynamicSecretId: string;
};
}
| {
name: QueueJobs.DynamicSecretPruning;
payload: {
dynamicSecretCfgId: string;
};
};
[QueueName.SecretReplication]: {
name: QueueJobs.SecretReplication;
payload: TSyncSecretsDTO;
};
[QueueName.SecretSync]: {
name: QueueJobs.SecretSync;
payload: TSyncSecretsDTO;
};
[QueueName.PkiSync]:
| {
name: QueueJobs.PkiSyncSyncCertificates;
payload: TQueuePkiSyncSyncCertificatesByIdDTO;
}
| {
name: QueueJobs.PkiSyncImportCertificates;
payload: TQueuePkiSyncImportCertificatesByIdDTO;
}
| {
name: QueueJobs.PkiSyncRemoveCertificates;
payload: TQueuePkiSyncRemoveCertificatesByIdDTO;
}
| {
name: QueueJobs.PkiSyncLinkMatchingCertificates;
payload: TQueuePkiSyncLinkMatchingCertificatesDTO;
}
| {
name: QueueJobs.PkiSyncReconcileFilters;
payload: TQueuePkiSyncReconcileFiltersDTO;
};
[QueueName.PkiSyncHealthCheck]: {
name: QueueJobs.PkiSyncRunHealthCheck;
payload: TQueuePkiSyncRunHealthCheckByIdDTO;
};
[QueueName.ProjectV3Migration]: {
name: QueueJobs.ProjectV3Migration;
payload: { projectId: string };
};
[QueueName.FolderTreeCheckpoint]: {
name: QueueJobs.CreateFolderTreeCheckpoint;
payload: {
envId: string;
};
};
[QueueName.ImportSecretsFromExternalSource]: {
name: QueueJobs.ImportSecretsFromExternalSource;
payload: {
orgId: string;
actorId: string;
actorEmail: string;
importType: ExternalPlatforms;
data: {
iv: string;
tag: string;
ciphertext: string;
algorithm: SecretEncryptionAlgo;
encoding: SecretKeyEncoding;
};
};
};
[QueueName.VaultImportSideEffects]: {
name: QueueJobs.VaultImportSideEffects;
payload: TVaultImportSideEffectsJobPayload;
};
[QueueName.AppConnectionSecretSync]:
| {
name: QueueJobs.SecretSyncSyncSecrets;
payload: TQueueSecretSyncSyncSecretsByIdDTO;
}
| {
name: QueueJobs.SecretSyncImportSecrets;
payload: TQueueSecretSyncImportSecretsByIdDTO;
}
| {
name: QueueJobs.SecretSyncRemoveSecrets;
payload: TQueueSecretSyncRemoveSecretsByIdDTO;
}
| {
name: QueueJobs.SecretSyncSendActionFailedNotifications;
payload: TQueueSendSecretSyncActionFailedNotificationsDTO;
}
| {
name: QueueJobs.DailySecretSyncRetry;
payload: undefined;
};
[QueueName.SecretRotationV2]:
| {
name: QueueJobs.SecretRotationV2QueueRotations;
payload: undefined;
}
| {
name: QueueJobs.SecretRotationV2SendNotification;
payload: TSecretRotationSendNotificationJobPayload;
};
[QueueName.SecretRotationV2RotateSecrets]: {
name: QueueJobs.SecretRotationV2RotateSecrets;
payload: TSecretRotationRotateSecretsJobPayload;
};
[QueueName.PamCredentialRotation]: {
name: QueueJobs.PamCredentialRotationQueueRotations;
payload: undefined;
};
[QueueName.PamCredentialRotationRotate]: {
name: QueueJobs.PamCredentialRotationRotate;
payload: { accountId: string };
};
[QueueName.PamHeartbeat]: {
name: QueueJobs.PamHeartbeatQueueChecks;
payload: undefined;
};
[QueueName.PamHeartbeatCheck]: {
name: QueueJobs.PamHeartbeatCheck;
payload: { accountId: string };
};
[QueueName.InvalidateCache]: {
name: QueueJobs.InvalidateCache;
payload: {
data: {
type: CacheType;
};
};
};
[QueueName.SecretScanningV2]:
| {
name: QueueJobs.SecretScanningV2FullScan;
payload: TQueueSecretScanningDataSourceFullScan;
}
| {
name: QueueJobs.SecretScanningV2DiffScan;
payload: TQueueSecretScanningResourceDiffScan;
}
| {
name: QueueJobs.SecretScanningV2SendNotification;
payload: TQueueSecretScanningSendNotification;
}
| {
name: QueueJobs.SecretScanningV2DataSourceEvent;
payload: TQueueSecretScanningDataSourceEvent;
};
[QueueName.SecretScanningV2FullScan]: {
name: QueueJobs.SecretScanningV2FullScan;
payload: TQueueSecretScanningDataSourceFullScan;
};
[QueueName.SecretScanningV2RealtimeScan]: {
name: QueueJobs.SecretScanningV2DiffScan;
payload: TQueueSecretScanningResourceDiffScanPayload;
};
[QueueName.CaLifecycle]: {
name: QueueJobs.CaOrderCertificateForSubscriber;
payload: {
subscriberId: string;
caType: CaType;
};
};
[QueueName.CertificateIssuance]: {
name: QueueJobs.CaIssueCertificateFromProfile;
payload: {
certificateId: string;
profileId: string;
caId: string;
commonName?: string;
altNames?: Array<{ type: string; value: string }>;
ttl: string;
signatureAlgorithm: string;
keyAlgorithm: string;
keyUsages?: string[];
extendedKeyUsages?: string[];
isRenewal?: boolean;
originalCertificateId?: string;
certificateRequestId?: string;
csr?: string;
organization?: string;
organizationalUnit?: string;
country?: string;
state?: string;
locality?: string;
basicConstraints?: { isCA: boolean; pathLength?: number | null } | null;
};
};
[QueueName.PkiSubscriber]: {
name: QueueJobs.PkiSubscriberDailyAutoRenewal;
payload: undefined;
};
[QueueName.UserNotification]: {
name: QueueJobs.UserNotification;
payload: { notifications: TCreateUserNotificationDTO[] };
};
[QueueName.AlertDispatch]: {
name: QueueJobs.AlertDispatch;
payload: { alertId: string; scheduledAt: string };
};
[QueueName.EventOutboxFlush]: {
name: QueueJobs.EventOutboxFlush;
payload: { consumer: string };
};
[QueueName.AuditReportGeneration]: {
name: QueueJobs.GenerateAuditReport;
payload: { auditReportId: string };
};
[QueueName.PamSessionExpiration]: {
name: QueueJobs.PamSessionExpiration;
payload: { sessionId: string };
};
[QueueName.PamDiscoveryScan]: {
name: QueueJobs.PamDiscoverySourceScan;
payload: { sourceId: string; triggeredBy: string };
};
[QueueName.PkiAcmeChallengeValidation]: {
name: QueueJobs.PkiAcmeChallengeValidation;
payload: { challengeId: string };
};
[QueueName.PkiDiscoveryScan]:
| {
name: QueueJobs.PkiDiscoveryRunScan;
payload: { discoveryId: string };
}
| {
name: QueueJobs.PkiDiscoveryScheduledScan;
payload: undefined;
};
[QueueName.PkiDiscoveryRescan]: {
name: QueueJobs.PkiDiscoveryRescanInstallation;
payload: { installationId: string };
};
[QueueName.AppConnectionCredentialRotation]:
| {
name: QueueJobs.AppConnectionCredentialRotationQueueRotations;
payload: undefined;
}
| {
name: QueueJobs.AppConnectionCredentialRotationSendNotification;
payload: TAppConnectionCredentialRotationSendNotificationJobPayload;
};
[QueueName.AppConnectionCredentialRotationRotate]: {
name: QueueJobs.AppConnectionCredentialRotationRotate;
payload: TAppConnectionCredentialRotationRotateJobPayload;
};
[QueueName.AuditLogClickHouseBatch]: {
name: QueueJobs.AuditLogClickHouseBatch;
payload: undefined;
};
[QueueName.AuditLogStreamOutbox]: {
name: QueueJobs.AuditLogStreamFlush;
payload: TAuditLogStreamFlushJobData;
};
[QueueName.AuditLogCollapsedFlush]: {
name: QueueJobs.AuditLogCollapsedFlush;
payload: TAuditLogCollapsedFlushJobData;
};
[QueueName.CaAutoRenewal]:
| {
name: QueueJobs.CaDailyAutoRenewal;
payload: undefined;
}
| {
name: QueueJobs.CaVenafiInstall;
payload: { caId: string; maxPathLength?: number };
}
| {
name: QueueJobs.CaAdcsInstall;
payload: { caId: string; maxPathLength?: number };
}
| {
name: QueueJobs.CaNativeAdcsInstall;
payload: { caId: string; maxPathLength?: number };
};
[QueueName.ProjectHardDelete]: {
name: QueueJobs.ProjectHardDelete;
payload: { projectId: string };
};
[QueueName.ProjectEnvHardDelete]: {
name: QueueJobs.ProjectEnvHardDelete;
payload: { envId: string; projectId: string };
};
[QueueName.SignerAutoRenewal]: {
name: QueueJobs.SignerDailyAutoRenewal;
payload: undefined;
};
[QueueName.SecretBlindIndexMigration]: {
name: QueueJobs.SecretBlindIndexMigration;
// `scope` is absent on jobs queued before the org-wide walk shipped.
payload: { scope: "org"; orgId: string } | { scope: "project"; projectId: string } | { projectId: string };
};
[QueueName.UsageEvent]: {
name: QueueJobs.UsageEvent;
payload: { orgId: string; dimensionKey: string };
};
[QueueName.IntegrationDeprecationNotice]: {
name: QueueJobs.SendIntegrationDeprecationNotice;
// period is a YYYY-MM stamp computed once by the cron tick so every retry of the same fire is deduped alike
payload: { orgId: string; period: string };
};
[QueueName.LegacyPkiDeprecationNotice]: {
name: QueueJobs.SendLegacyPkiDeprecationNotice;
// period is a YYYY-MM stamp computed once by the cron tick so every retry of the same fire is deduped alike
payload: { orgId: string; period: string };
};
};
const SECRET_SCANNING_QUEUES = [
QueueName.SecretScanningV2,
QueueName.SecretScanningV2FullScan,
QueueName.SecretScanningV2RealtimeScan,
QueueName.SecretFullRepoScan,
QueueName.SecretPushEventScan
];
const isQueueEnabled = (name: QueueName) => {
const appCfg = getConfig();
return SECRET_SCANNING_QUEUES.includes(name)
? appCfg.isSecretScanningRunModeEnabled
: appCfg.isGeneralWorkerRunModeEnabled;
};
export type TQueueServiceFactory = {
start: <T extends QueueName>(
name: T,
jobFn: (
job: Job<TQueueJobTypes[T]["payload"], void, TQueueJobTypes[T]["name"]>,
token?: string,
signal?: AbortSignal
) => Promise<void>,
queueSettings?: Omit<QueueOptions, "connection"> &
Pick<WorkerOptions, "concurrency" | "limiter" | "maxStalledCount">
) => void;
listen: <
T extends QueueName,
U extends keyof WorkerListener<TQueueJobTypes[T]["payload"], void, TQueueJobTypes[T]["name"]>
>(
name: T,
event: U,
listener: WorkerListener<TQueueJobTypes[T]["payload"], void, TQueueJobTypes[T]["name"]>[U]
) => void;
queue: <T extends QueueName>(
name: T,
job: TQueueJobTypes[T]["name"],
data: TQueueJobTypes[T]["payload"],
opts: TQueueOptions
) => Promise<void>;
shutdown: () => Promise<void>;
// @deprecated Use removeJobScheduler instead.
stopRepeatableJob: <T extends QueueName>(
name: T,
job: TQueueJobTypes[T]["name"],
repeatOpt: RepeatOptions,
jobId?: string
) => Promise<boolean | undefined>;
// @deprecated Use stopJobById for delayed jobs. Use removeJobScheduler for schedulers.
stopRepeatableJobByJobId: <T extends QueueName>(name: T, jobId: string) => Promise<boolean>;
// @deprecated Use removeJobScheduler instead.
stopRepeatableJobByKey: <T extends QueueName>(name: T, repeatJobKey: string) => Promise<boolean>;
clearQueue: (name: QueueName) => Promise<void>;
stopJobById: <T extends QueueName>(name: T, jobId: string) => Promise<void | undefined>;
cancelActiveJob: <T extends QueueName>(name: T, jobId: string, reason?: string) => boolean;
// @deprecated Use getJobSchedulers instead.
getRepeatableJobs: (
name: QueueName,
startOffset?: number,
endOffset?: number
) => Promise<{ key: string; name: string; id?: string | null }[]>;
getDelayedJobs: (
name: QueueName,
startOffset?: number,
endOffset?: number
) => Promise<{ delay: number; timestamp: number; repeatJobKey?: string; data?: unknown }[]>;
upsertJobScheduler: <T extends QueueName>(
name: T,
schedulerId: string,
repeatConfig: { pattern?: string; every?: number },
jobTemplate?: {
name?: TQueueJobTypes[T]["name"];
data?: TQueueJobTypes[T]["payload"];
opts?: {
removeOnComplete?: boolean | { count: number };
removeOnFail?: boolean | { count: number };
delay?: number;
};
}
) => Promise<void>;
removeJobScheduler: <T extends QueueName>(name: T, schedulerId: string) => Promise<void>;
getJobSchedulers: (name: QueueName, start?: number, end?: number) => Promise<JobSchedulerJson[]>;
getJob: <T extends QueueName>(
name: T,
jobId: string
) => Promise<Job<TQueueJobTypes[T]["payload"], void, string> | undefined>;
};
export const queueServiceFactory = (redisCfg: TRedisConfigKeys): TQueueServiceFactory => {
const isClusterMode = Boolean(redisCfg?.REDIS_CLUSTER_HOSTS);
const connection = buildRedisFromConfig(redisCfg, "queue");
const queueContainer: Partial<Record<QueueName, Queue<TQueueJobTypes[QueueName]["payload"], void, string>>> = {};
const workerContainer: Partial<
Record<QueueName, Worker<TQueueJobTypes[QueueName]["payload"], void, TQueueJobTypes[QueueName]["name"]>>
> = {};
// One logger for every worker, so an outage hitting all of them still logs one line per error per window.
const workerErrorLogger = createThrottledRedisErrorLogger("queue-workers", "Queue worker error");
// Observable gauge for queue depth. The SDK invokes the callback on each export (every 30s for OTLP
// push, on each scrape for Prometheus). Iterates only initialized queues in queueContainer; one
// snapshot covers all ~30 named queues. Failures are swallowed because metrics must never crash the app.
const QUEUE_DEPTH_STATES = ["waiting", "active", "delayed", "failed"] as const;
const queueDepthGauge = resolveCoreMeter().createObservableGauge("infisical.queue.depth", {
description: "Number of jobs in each queue state (waiting, active, delayed, failed)",
unit: "{job}"
});
queueDepthGauge.addCallback(async (observableResult) => {
if (!getConfig().OTEL_TELEMETRY_COLLECTION_ENABLED) return;
await Promise.allSettled(
Object.entries(queueContainer).map(async ([name, q]) => {
if (!q) return;
try {
const counts = await q.getJobCounts(...QUEUE_DEPTH_STATES);
Object.entries(counts).forEach(([state, count]) => {
if (typeof count !== "number") return;
observableResult.observe(count, { "queue.name": name, "queue.state": state });
});
} catch (err) {
logger.warn({ err, queue: name }, `queue.depth gauge: getJobCounts failed [queue=${name}]`);
}
})
);
});
// Remove orphaned job schedulers left in Redis by deleted queues.
// Queues migrated to the cronJob system (cron-job.ts) are listed here so their
// BullMQ schedulers and pending jobs are cleaned up on first boot of the new image.
//
// Gated to general-workers because every name below belongs to that fleet, and obliterate() is
// called with force, which deletes active jobs too. Reaping is the consuming pod's job: an API
// pod has no worker on these queues and must not clear work another pod is running.
void (async () => {
if (!getConfig().isGeneralWorkerRunModeEnabled) return;
const staleQueueNames = [
"queue-internal-recovery",
"queue-internal-reconciliation",
"secret-rotation",
// Legacy per-log audit queue, replaced by the unified Redis ingest stream. The compatibility
// shim that re-routed its jobs has been removed; obliterate any residue left in Redis.
"audit-log",
// Queues replaced by cronJobFactory (src/lib/cron/cron-job.ts)
"daily-resource-cleanup",
"frequent-resource-cleanup",
"daily-reminders",
"secret-reminder-migration",
"health-alert",
"certificate-cleanup",
"pki-sync-cleanup",
"pam-account-rotation",
"pam-session-ai-summary",
"daily-pki-alert-v2-processing",
"daily-expiring-pki-item-alert",
"telemtry-self-hosted-stats", // note: typo from original enum value
"telemetry-aggregated-events",
"certificate-v3-auto-renewal",
"ca-crl-rotation",
"digicert-order-polling"
];
await Promise.allSettled(
staleQueueNames.map(async (name) => {
const staleQueue = new Queue(name, {
prefix: isClusterMode ? `{${name}}` : undefined,
connection
});
await staleQueue.obliterate({ force: true });
await staleQueue.close();
logger.info({ queue: name }, "Cleaned up orphaned internal queue from Redis");
})
);
// Remove stale BullMQ schedulers for queues migrated to cronJobFactory.
// Without this, the old scheduler keeps firing via the active Worker, causing double execution.
const staleSchedulersInActiveQueues: Array<{ queueName: string; schedulerId: string }> = [
{ queueName: "pki-subscriber", schedulerId: `${JOB_SCHEDULER_PREFIX}:pki-subscriber` },
{ queueName: "pki-discovery-scan", schedulerId: `${JOB_SCHEDULER_PREFIX}:pki-discovery-scheduled-scan` },
{ queueName: "secret-rotation-v2", schedulerId: `${JOB_SCHEDULER_PREFIX}:secret-rotation-v2-cron` },
{
queueName: "app-connection-credential-rotation",
schedulerId: `${JOB_SCHEDULER_PREFIX}:app-connection-credential-rotation-cron`
},
{ queueName: "app-connection-secret-sync", schedulerId: `${JOB_SCHEDULER_PREFIX}:daily-secret-sync-retry-job` },
{ queueName: "ca-auto-renewal", schedulerId: `${JOB_SCHEDULER_PREFIX}:ca-daily-auto-renewal` }
];
await Promise.allSettled(
staleSchedulersInActiveQueues.map(async ({ queueName, schedulerId }) => {
const q = new Queue(queueName, {
prefix: isClusterMode ? `{${queueName}}` : undefined,
connection
});
try {
await q.removeJobScheduler(schedulerId);
logger.info({ queue: queueName, schedulerId }, "Removed orphaned job scheduler from active queue");
} catch (err) {
logger.warn(
{ err, queue: queueName, schedulerId },
"Failed to remove orphaned job scheduler from active queue"
);
} finally {
await q.close();
}
})
);
})();
const start: TQueueServiceFactory["start"] = (name, jobFn, queueSettings) => {
if (queueContainer[name]) {
throw new Error(`${name} queue is already initialized`);
}
const fipsSettings = crypto.isFipsModeEnabled() ? { settings: { repeatKeyHashAlgorithm: "sha256" as const } } : {};
// The Queue (producer) is created regardless of run mode — only the Worker (consumer) is
// gated below. A pod that doesn't consume a queue must still be able to enqueue onto it, or
// splitting the fleet by run mode silently drops every job destined for another pod's worker.
queueContainer[name] = new Queue(name as string, {
prefix: isClusterMode ? `{${name}}` : undefined,
...queueSettings,
...fipsSettings,
connection
});
if (!isQueueEnabled(name)) {
return;
}
const worker = new Worker(name, jobFn, {
prefix: isClusterMode ? `{${name}}` : undefined,
...fipsSettings,
...queueSettings,
// Enable BullMQ's built-in per-minute completion/failure tracking in Redis. Survives pod restarts
// (OTel cumulative counters reset), useful as a fallback when Prometheus retention is short.
metrics: { maxDataPoints: MetricsTime.ONE_WEEK * 2 },
connection
});
// Cross-cutting metric emission for every queue. Single registration covers all ~30 named queues
// since BullMQ Worker is an EventEmitter and we attach the listeners here.
worker.on("completed", (job) => {
const baseAttrs = { "queue.name": name, "job.name": job.name } as Record<string, string>;
queueJobCounter.add(1, { ...baseAttrs, outcome: "completed" });
logger.debug({ queue: name, job: job.name, jobId: job.id }, "Queue job completed");
if (typeof job.processedOn === "number" && typeof job.timestamp === "number") {
const durationMs = Date.now() - job.processedOn;
queueJobDurationHistogram.record(durationMs / 1000, { ...baseAttrs, outcome: "completed" });
// Wait = (worker pickup) - (enqueue time + intentional delay). Subtracting opts.delay is what
// makes this measure queue contention, not scheduling
const intentionalDelayMs = job.opts.delay ?? 0;
const waitMs = Math.max(0, job.processedOn - job.timestamp - intentionalDelayMs);
queueJobWaitHistogram.record(waitMs / 1000, baseAttrs);
}
});
worker.on("failed", (job, err) => {
const baseAttrs = {
"queue.name": name,
"job.name": job?.name ?? "unknown"
} as Record<string, string>;
queueJobCounter.add(1, { ...baseAttrs, outcome: "failed" });
// Skip duration on framework-level failures (job hydration failed before start). Emitting 0
// would pollute the histogram with phantom zero-duration points.
if (job && typeof job.processedOn === "number") {
const durationMs = Date.now() - job.processedOn;
queueJobDurationHistogram.record(durationMs / 1000, { ...baseAttrs, outcome: "failed" });
}
const errorType = classifyError(err);
const attemptsExhausted = !!job && job.attemptsMade >= (job.opts.attempts || 1);
queueJobFailureCounter.add(1, {
...baseAttrs,
"error.type": errorType,
"attempts.exhausted": attemptsExhausted ? "true" : "false"
});
logger.error(
err,
`Queue job failed [queue=${name}] [job=${job?.name ?? "unknown"}] [jobId=${job?.id ?? "unknown"}] [attempt=${
job?.attemptsMade ?? 0
}] [attemptsExhausted=${attemptsExhausted}]`
);
});
// Stalling means the worker stopped heartbeating and BullMQ handed the job to someone else, so
// it is both rare and always worth seeing. It stays at warn rather than debug for that reason.
worker.on("stalled", (jobId) => {
queueStalledCounter.add(1, { "queue.name": name });
logger.warn({ queue: name, jobId }, `Queue job stalled and was requeued [queue=${name}] [jobId=${jobId}]`);
});
worker.on("active", (job) => {
logger.debug({ queue: name, job: job.name, jobId: job.id }, "Queue job picked up by a worker");
});
// The worker's own blocking connection is a duplicate with no listeners, and BullMQ only reports its
// errors (and run-loop errors) through this event.
worker.on("error", (err) => workerErrorLogger.log(err, undefined, { queue: name }));
workerContainer[name] = worker;
};
const listen: TQueueServiceFactory["listen"] = (name, event, listener) => {
if (!isQueueEnabled(name)) {
return;
}
const worker = workerContainer[name];
worker?.on(event, listener);
};
const queue: TQueueServiceFactory["queue"] = async (name, job, data, opts) => {
const q = queueContainer[name];
const { jobId, repeat } = opts;
const finalOptions = {
removeOnFail: true,
removeOnComplete: true,
...opts,
repeat: repeat ? { ...repeat, utc: true } : undefined,
jobId
};
const addedJob = await q?.add(job, data, finalOptions);
// Only when the job really exists. `q` is undefined only where QUEUE_WORKERS_ENABLED is false,
// and a pod in that mode neither produces nor consumes, so a no-op there is expected rather
// than a dropped job worth reporting.
if (addedJob) logger.debug({ queue: name, job, jobId: addedJob.id }, "Queue job enqueued");
};
const stopRepeatableJob: TQueueServiceFactory["stopRepeatableJob"] = async (name, job, repeatOpt, jobId) => {
const q = queueContainer[name];
if (q) {
return q.removeRepeatable(job, repeatOpt, jobId);
}
};
const getRepeatableJobs: TQueueServiceFactory["getRepeatableJobs"] = (name, startOffset, endOffset) => {
const q = queueContainer[name];
if (!q) throw new Error(`Queue '${name}' not initialized`);
return q.getRepeatableJobs(startOffset, endOffset);
};
const getDelayedJobs: TQueueServiceFactory["getDelayedJobs"] = (name, startOffset, endOffset) => {
const q = queueContainer[name];
if (!q) throw new Error(`Queue '${name}' not initialized`);
return q.getDelayed(startOffset, endOffset);
};
const stopRepeatableJobByJobId: TQueueServiceFactory["stopRepeatableJobByJobId"] = async (name, jobId) => {
const q = queueContainer[name];
if (!q) {
return true;
}
const job = await q.getJob(jobId);
if (!job?.repeatJobKey) {
return true;
}
await job.remove();
return q.removeRepeatableByKey(job.repeatJobKey);
};
const stopRepeatableJobByKey: TQueueServiceFactory["stopRepeatableJobByKey"] = async (name, repeatJobKey) => {
const q = queueContainer[name];
if (!q) {
return true;
}
return q?.removeRepeatableByKey(repeatJobKey);
};
const stopJobById: TQueueServiceFactory["stopJobById"] = async (name, jobId) => {
const q = queueContainer[name];
const job = await q?.getJob(jobId);
return job?.remove().catch(() => undefined);
};
const cancelActiveJob: TQueueServiceFactory["cancelActiveJob"] = (name, jobId, reason) => {
const w = workerContainer[name];
if (!w) return false;
return w.cancelJob(jobId, reason);
};
const clearQueue: TQueueServiceFactory["clearQueue"] = async (name) => {
const q = queueContainer[name];
await q?.drain();
};
const shutdown: TQueueServiceFactory["shutdown"] = async () => {
await Promise.all(Object.values(workerContainer).map((worker) => worker.close()));
};
const upsertJobScheduler: TQueueServiceFactory["upsertJobScheduler"] = async (
name,
schedulerId,
repeatConfig,
jobTemplate
) => {
const q = queueContainer[name];
if (!q) throw new Error(`Queue '${name}' not initialized`);
// Remove legacy repeatable jobs that don't use the job scheduler prefix.
// This prevents duplicate execution after migrating from queue.add({repeat}) to upsertJobScheduler.
try {
const repeatableJobs = await q.getRepeatableJobs();
const legacyJobs = repeatableJobs.filter((job) => !job.key.startsWith(JOB_SCHEDULER_PREFIX));
await Promise.all(
legacyJobs.map((job) =>
q.removeRepeatableByKey(job.key).then(() => {
logger.info({ queue: name, key: job.key }, "Removed legacy repeatable job");
})
)
);
} catch (err) {
logger.error(err, `Failed to clean up legacy repeatable jobs for queue '${name}'`);
}
// Remove orphaned delayed jobs left behind by legacy repeatable schedules.
// removeRepeatableByKey cleans up the ZSET entry and its delayed job, but workers may
// have already processed the old delayed job and re-created the next iteration before
// the ZSET entry was removed. Runs before and after upsert to handle the race window.
const $removeLegacyDelayedJobs = async () => {
const delayedJobs = await q.getDelayed();
const legacyDelayedJobs = delayedJobs.filter(
(job) => job?.id?.startsWith("repeat:") && !job.id.includes(JOB_SCHEDULER_PREFIX)
);
await Promise.all(
legacyDelayedJobs.map((job) =>
job
.remove()
.then(() => {
logger.info({ queue: name, jobId: job.id }, "Removed orphaned legacy delayed job");
})
.catch((removeErr: unknown) => {
logger.warn(
{ queue: name, jobId: job.id, error: removeErr },
"Failed to remove orphaned legacy delayed job"
);
})
)
);
};
try {
await $removeLegacyDelayedJobs();
} catch (err) {
logger.error(err, `Failed to clean up orphaned legacy delayed jobs for queue '${name}'`);
}
await q.upsertJobScheduler(
schedulerId,
{ ...repeatConfig, utc: true },
{
name: jobTemplate?.name ?? schedulerId,
data: jobTemplate?.data,
opts: {
removeOnComplete: true,
removeOnFail: true,
...jobTemplate?.opts
}
}
);
try {
await $removeLegacyDelayedJobs();
} catch (err) {
logger.error(err, `Failed second-pass cleanup of orphaned legacy delayed jobs for queue '${name}'`);
}
};
const removeJobScheduler: TQueueServiceFactory["removeJobScheduler"] = async (name, schedulerId) => {
const q = queueContainer[name];
if (!q) throw new Error(`Queue '${name}' not initialized`);
await q.removeJobScheduler(schedulerId);
};
const getJobSchedulers: TQueueServiceFactory["getJobSchedulers"] = async (name, startOffset, endOffset) => {
const q = queueContainer[name];
if (!q) throw new Error(`Queue '${name}' not initialized`);
return q.getJobSchedulers(startOffset, endOffset);
};
const getJob: TQueueServiceFactory["getJob"] = async (name, jobId) => {
const q = queueContainer[name];
if (!q) return undefined;
const job = await q.getJob(jobId);
return job ?? undefined;
};
return {
start,
listen,
queue,
shutdown,
stopRepeatableJob,
stopRepeatableJobByJobId,
stopRepeatableJobByKey,
clearQueue,
stopJobById,
cancelActiveJob,
getRepeatableJobs,
getDelayedJobs,
upsertJobScheduler,
removeJobScheduler,
getJobSchedulers,
getJob
};
};