Skip to content

Infisical is the open-source platform for securing secrets, certificates, access, and AI Agents.

43.4 KB
Raw Blame
import {
  Job,
  JobSchedulerJson,
  MetricsTime,
  Queue,
  QueueOptions,
  RepeatOptions,
  Worker,
  WorkerListener,
  WorkerOptions
} from "bullmq";

import { SecretEncryptionAlgo, SecretKeyEncoding } from "@app/db/schemas";
import { TAuditLogStreamFlushJobData } from "@app/ee/services/audit-log-stream-outbox/audit-log-stream-outbox-types";
import { TAuditLogCollapsedFlushJobData } from "@app/ee/services/audit-log/audit-log-types";
import {
  TSecretRotationRotateSecretsJobPayload,
  TSecretRotationSendNotificationJobPayload
} from "@app/ee/services/secret-rotation-v2/secret-rotation-v2-types";
import {
  TQueueSecretScanningDataSourceEvent,
  TQueueSecretScanningDataSourceFullScan,
  TQueueSecretScanningResourceDiffScan,
  TQueueSecretScanningResourceDiffScanPayload,
  TQueueSecretScanningSendNotification
} from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-types";
import {
  TScanFullRepoEventPayload,
  TScanPushEventPayload
} from "@app/ee/services/secret-scanning/secret-scanning-queue/secret-scanning-queue-types";
import { getConfig } from "@app/lib/config/env";
import { buildRedisFromConfig, createThrottledRedisErrorLogger, TRedisConfigKeys } from "@app/lib/config/redis";
import { crypto } from "@app/lib/crypto";
import { classifyError } from "@app/lib/errors/classify";
import { logger } from "@app/lib/logger";
import {
  queueJobCounter,
  queueJobDurationHistogram,
  queueJobFailureCounter,
  queueJobWaitHistogram,
  queueStalledCounter,
  resolveCoreMeter
} from "@app/lib/telemetry/metrics";
import {
  TAppConnectionCredentialRotationRotateJobPayload,
  TAppConnectionCredentialRotationSendNotificationJobPayload
} from "@app/services/app-connection/credential-rotation/app-connection-credential-rotation-types";
import { CaType } from "@app/services/certificate-authority/certificate-authority-enums";
import {
  ExternalPlatforms,
  TVaultImportSideEffectsJobPayload
} from "@app/services/external-migration/external-migration-types";
import { TCreateUserNotificationDTO } from "@app/services/notification/notification-types";
import { PkiAlertEventType } from "@app/services/pki-alert-v2/pki-alert-v2-types";
import {
  TQueuePkiSyncImportCertificatesByIdDTO,
  TQueuePkiSyncLinkMatchingCertificatesDTO,
  TQueuePkiSyncReconcileFiltersDTO,
  TQueuePkiSyncRemoveCertificatesByIdDTO,
  TQueuePkiSyncRunHealthCheckByIdDTO,
  TQueuePkiSyncSyncCertificatesByIdDTO
} from "@app/services/pki-sync/pki-sync-types";
import {
  TQueueSecretSyncImportSecretsByIdDTO,
  TQueueSecretSyncRemoveSecretsByIdDTO,
  TQueueSecretSyncSyncSecretsByIdDTO,
  TQueueSendSecretSyncActionFailedNotificationsDTO
} from "@app/services/secret-sync/secret-sync-types";
import {
  TFailedIntegrationSyncEmailsPayload,
  TIntegrationSyncPayload,
  TSyncSecretsDTO
} from "@app/services/secret/secret-types";
import { CacheType } from "@app/services/super-admin/super-admin-types";
import { TWebhookPayloads } from "@app/services/webhook/webhook-types";

// Scheduler IDs are prefixed so they never collide with legacy repeatable-job
// iteration IDs that may still be pending in Redis after the migration.
// Bump the version if a future migration needs the same trick.
export const JOB_SCHEDULER_PREFIX = "jsv1";

export enum QueueName {
  SecretReminder = "secret-reminder",
  // TODO(akhilmhdh): This will get removed later. For now this is kept to stop the repeatable queue
  AuditLogPrune = "audit-log-prune",
  PkiAlertV2Event = "pki-alert-v2-event",
  PkiSubscriber = "pki-subscriber",
  IntegrationSync = "sync-integrations",
  SecretWebhook = "secret-webhook",
  SecretFullRepoScan = "secret-full-repo-scan",
  SecretPushEventScan = "secret-push-event-scan",
  UpgradeProjectToGhost = "upgrade-project-to-ghost",
  DynamicSecretRevocation = "dynamic-secret-revocation",
  DynamicSecretLeaseRevocationFailedEmail = "dynamic-secret-lease-revocation-failed-email",
  CaLifecycle = "ca-lifecycle", // parent queue to ca-order-certificate-for-subscriber
  CertificateIssuance = "certificate-issuance",
  SecretReplication = "secret-replication",
  SecretSync = "secret-sync", // parent queue to push integration sync, webhook, and secret replication
  PkiSync = "pki-sync",
  PkiSyncHealthCheck = "pki-sync-health-check",
  ProjectV3Migration = "project-v3-migration",
  AccessTokenStatusUpdate = "access-token-status-update",
  ImportSecretsFromExternalSource = "import-secrets-from-external-source",
  VaultImportSideEffects = "vault-import-side-effects",
  AppConnectionSecretSync = "app-connection-secret-sync",
  SecretRotationV2 = "secret-rotation-v2",
  SecretRotationV2RotateSecrets = "secret-rotation-v2-rotate-secrets",
  PamCredentialRotation = "pam-credential-rotation",
  PamCredentialRotationRotate = "pam-credential-rotation-rotate",
  PamHeartbeat = "pam-heartbeat",
  PamHeartbeatCheck = "pam-heartbeat-check",
  FolderTreeCheckpoint = "folder-tree-checkpoint",
  InvalidateCache = "invalidate-cache",
  SecretScanningV2 = "secret-scanning-v2",
  SecretScanningV2FullScan = "secret-scanning-v2-full-scan",
  SecretScanningV2RealtimeScan = "secret-scanning-v2-realtime-scan",
  UserNotification = "user-notification",
  AlertDispatch = "alert-dispatch",
  EventOutboxFlush = "event-outbox-flush",
  AuditReportGeneration = "audit-report-generation",
  PamSessionExpiration = "pam-session-expiration",
  PamDiscoveryScan = "pam-discovery-scan",
  PkiAcmeChallengeValidation = "pki-acme-challenge-validation",
  PkiDiscoveryScan = "pki-discovery-scan",
  PkiDiscoveryRescan = "pki-discovery-rescan",
  AppConnectionCredentialRotation = "app-connection-credential-rotation",
  AppConnectionCredentialRotationRotate = "app-connection-credential-rotation-rotate",
  AuditLogClickHouseBatch = "audit-log-clickhouse-batch",
  AuditLogStreamOutbox = "audit-log-stream-outbox",
  AuditLogCollapsedFlush = "audit-log-collapsed-flush",
  CaAutoRenewal = "ca-auto-renewal",
  ProjectHardDelete = "project-hard-delete",
  ProjectEnvHardDelete = "project-env-hard-delete",
  SignerAutoRenewal = "signer-auto-renewal",
  SecretBlindIndexMigration = "secret-blind-index-migration",
  UsageEvent = "usage-event",
  IntegrationDeprecationNotice = "integration-deprecation-notice",
  LegacyPkiDeprecationNotice = "legacy-pki-deprecation-notice"
}

export enum QueueJobs {
  SecretReminder = "secret-reminder-job",
  // TODO(akhilmhdh): This will get removed later. For now this is kept to stop the repeatable queue
  AuditLogPrune = "audit-log-prune-job",
  DailyResourceCleanUp = "daily-resource-cleanup-job",
  FrequentResourceCleanUp = "frequent-resource-cleanup-job",
  DailyExpiringPkiItemAlert = "daily-expiring-pki-item-alert",
  DailyPkiAlertV2Processing = "daily-pki-alert-v2-processing",
  PkiAlertV2ProcessEvent = "pki-alert-v2-process-event",
  PkiSyncCleanup = "pki-sync-cleanup-job",
  SecWebhook = "secret-webhook-trigger",
  TelemetryInstanceStats = "telemetry-self-hosted-stats",
  IntegrationSync = "secret-integration-pull",
  SendFailedIntegrationSyncEmails = "send-failed-integration-sync-emails",
  SecretScan = "secret-scan",
  UpgradeProjectToGhost = "upgrade-project-to-ghost-job",
  DynamicSecretRevocation = "dynamic-secret-revocation",
  DynamicSecretPruning = "dynamic-secret-pruning",
  CaCrlRotation = "ca-crl-rotation-job",
  SecretReplication = "secret-replication",
  SecretSync = "secret-sync", // parent queue to push integration sync, webhook, and secret replication
  PkiSync = "pki-sync",
  ProjectV3Migration = "project-v3-migration",
  IdentityAccessTokenStatusUpdate = "identity-access-token-status-update",
  ServiceTokenStatusUpdate = "service-token-status-update",
  ImportSecretsFromExternalSource = "import-secrets-from-external-source",
  VaultImportSideEffects = "vault-import-side-effects",
  SecretSyncSyncSecrets = "secret-sync-sync-secrets",
  SecretSyncImportSecrets = "secret-sync-import-secrets",
  SecretSyncRemoveSecrets = "secret-sync-remove-secrets",
  SecretSyncSendActionFailedNotifications = "secret-sync-send-action-failed-notifications",
  PkiSyncSyncCertificates = "pki-sync-sync-certificates",
  PkiSyncLinkMatchingCertificates = "pki-sync-link-matching-certificates",
  PkiSyncReconcileFilters = "pki-sync-reconcile-filters",
  PkiSyncImportCertificates = "pki-sync-import-certificates",
  PkiSyncRemoveCertificates = "pki-sync-remove-certificates",
  PkiSyncRunHealthCheck = "pki-sync-run-health-check",
  SecretRotationV2QueueRotations = "secret-rotation-v2-queue-rotations",
  SecretRotationV2RotateSecrets = "secret-rotation-v2-rotate-secrets",
  SecretRotationV2SendNotification = "secret-rotation-v2-send-notification",
  PamCredentialRotationQueueRotations = "pam-credential-rotation-queue-rotations",
  PamCredentialRotationRotate = "pam-credential-rotation-rotate",
  PamHeartbeatQueueChecks = "pam-heartbeat-queue-checks",
  PamHeartbeatCheck = "pam-heartbeat-check",
  CreateFolderTreeCheckpoint = "create-folder-tree-checkpoint",
  DynamicSecretLeaseRevocationFailedEmail = "dynamic-secret-lease-revocation-failed-email",
  InvalidateCache = "invalidate-cache",
  SecretScanningV2FullScan = "secret-scanning-v2-full-scan",
  // Kept as "diff-scan" while the queue moved to "realtime-scan": jobs already enqueued under this
  // name carry it, and the drain branch on QueueName.SecretScanningV2 matches on it.
  SecretScanningV2DiffScan = "secret-scanning-v2-diff-scan",
  SecretScanningV2SendNotification = "secret-scanning-v2-notification",
  SecretScanningV2DataSourceEvent = "secret-scanning-v2-data-source-event",
  CaOrderCertificateForSubscriber = "ca-order-certificate-for-subscriber",
  CaIssueCertificateFromProfile = "ca-issue-certificate-from-profile",
  PkiSubscriberDailyAutoRenewal = "pki-subscriber-daily-auto-renewal",
  TelemetryAggregatedEvents = "telemetry-aggregated-events",
  DailyReminders = "daily-reminders",
  SecretReminderMigration = "secret-reminder-migration",
  UserNotification = "user-notification-job",
  AlertDispatch = "alert-dispatch-job",
  EventOutboxFlush = "event-outbox-flush-job",
  GenerateAuditReport = "generate-audit-report-job",
  HealthAlert = "health-alert",
  CertificateV3DailyAutoRenewal = "certificate-v3-daily-auto-renewal",
  PamSessionExpiration = "pam-session-expiration",
  PamDiscoverySourceScan = "pam-discovery-source-scan",
  PkiAcmeChallengeValidation = "pki-acme-challenge-validation",
  PkiDiscoveryRunScan = "pki-discovery-run-scan",
  PkiDiscoveryScheduledScan = "pki-discovery-scheduled-scan",
  PkiDiscoveryRescanInstallation = "pki-discovery-rescan-installation",
  AppConnectionCredentialRotationQueueRotations = "app-connection-credential-rotation-queue-rotations",
  AppConnectionCredentialRotationRotate = "app-connection-credential-rotation-rotate",
  AppConnectionCredentialRotationSendNotification = "app-connection-credential-rotation-send-notification",
  AuditLogClickHouseBatch = "audit-log-clickhouse-batch-job",
  AuditLogStreamFlush = "audit-log-stream-flush",
  AuditLogCollapsedFlush = "audit-log-collapsed-flush-job",
  CaDailyAutoRenewal = "ca-daily-auto-renewal",
  CaVenafiInstall = "ca-venafi-install-job",
  CaAdcsInstall = "ca-adcs-install-job",
  CaNativeAdcsInstall = "ca-native-adcs-install-job",
  CertificateCleanup = "certificate-cleanup-job",
  DailySecretSyncRetry = "daily-secret-sync-retry-job",
  DigiCertOrderPolling = "digicert-order-polling-job",
  GoDaddyOrderPolling = "godaddy-order-polling-job",
  ProjectHardDelete = "project-hard-delete-job",
  ProjectEnvHardDelete = "project-env-hard-delete-job",
  SignerDailyAutoRenewal = "signer-daily-auto-renewal",
  SecretBlindIndexMigration = "secret-blind-index-migration",
  UsageEvent = "usage-event-job",
  SendIntegrationDeprecationNotice = "send-integration-deprecation-notice",
  SendLegacyPkiDeprecationNotice = "send-legacy-pki-deprecation-notice"
}

export enum JobState {
  NotFound = "not-found",
  Pending = "pending",
  Completed = "completed",
  Failed = "failed"
}

type BaseQueueOptions = {
  removeOnComplete?: boolean | { count: number } | { age: number };
  removeOnFail?: boolean | { count: number } | { age: number };
  attempts?: number;
  delay?: number;
  backoff?: {
    type: "exponential" | "fixed";
    delay: number;
  };
  // @deprecated Use upsertJobScheduler instead.
  repeat?: {
    pattern?: string;
    every?: number;
    // only works with every by bullmq
    immediately?: boolean;
    key: string;
    utc?: boolean;
  };
};

type DeduplicationOptions = {
  id: string;
  keepLastIfActive?: boolean;
  replace?: boolean;
  extend?: boolean;
  ttl?: number;
};

export type TQueueOptions = BaseQueueOptions &
  (
    | {
        jobId: string;
        deduplication?: undefined;
      }
    | {
        jobId?: undefined;
        deduplication: DeduplicationOptions;
      }
  );

export type TQueueJobTypes = {
  [QueueName.SecretReminder]: {
    payload: {
      projectId: string;
      secretId: string;
      repeatDays: number;
      note: string | undefined | null;
    };
    name: QueueJobs.SecretReminder;
  };
  [QueueName.PkiAlertV2Event]: {
    name: QueueJobs.PkiAlertV2ProcessEvent;
    payload: {
      certificateId: string;
      projectId: string;
      eventType: PkiAlertEventType;
    };
  };
  [QueueName.AuditLogPrune]: {
    name: QueueJobs.AuditLogPrune;
    payload: undefined;
  };
  [QueueName.SecretWebhook]: {
    name: QueueJobs.SecWebhook;
    payload: TWebhookPayloads;
  };

  [QueueName.AccessTokenStatusUpdate]:
    | {
        name: QueueJobs.IdentityAccessTokenStatusUpdate;
        payload: { identityAccessTokenId: string; numberOfUses: number };
      }
    | {
        name: QueueJobs.ServiceTokenStatusUpdate;
        payload: { serviceTokenId: string };
      };

  [QueueName.IntegrationSync]:
    | {
        name: QueueJobs.IntegrationSync;
        payload: TIntegrationSyncPayload;
      }
    | {
        name: QueueJobs.SendFailedIntegrationSyncEmails;
        payload: TFailedIntegrationSyncEmailsPayload;
      };
  [QueueName.SecretFullRepoScan]: {
    name: QueueJobs.SecretScan;
    payload: TScanFullRepoEventPayload;
  };
  [QueueName.SecretPushEventScan]: { name: QueueJobs.SecretScan; payload: TScanPushEventPayload };
  [QueueName.UpgradeProjectToGhost]: {
    name: QueueJobs.UpgradeProjectToGhost;
    payload: {
      projectId: string;
      startedByUserId: string;
      encryptedPrivateKey: {
        encryptedKey: string;
        encryptedKeyIv: string;
        encryptedKeyTag: string;
        keyEncoding: SecretKeyEncoding;
      };
    };
  };
  [QueueName.DynamicSecretLeaseRevocationFailedEmail]: {
    name: QueueJobs.DynamicSecretLeaseRevocationFailedEmail;
    payload: {
      leaseId: string;
    };
  };
  [QueueName.DynamicSecretRevocation]:
    | {
        name: QueueJobs.DynamicSecretRevocation;
        payload: {
          isRetry?: boolean;
          leaseId: string;
          dynamicSecretId: string;
        };
      }
    | {
        name: QueueJobs.DynamicSecretPruning;
        payload: {
          dynamicSecretCfgId: string;
        };
      };
  [QueueName.SecretReplication]: {
    name: QueueJobs.SecretReplication;
    payload: TSyncSecretsDTO;
  };
  [QueueName.SecretSync]: {
    name: QueueJobs.SecretSync;
    payload: TSyncSecretsDTO;
  };
  [QueueName.PkiSync]:
    | {
        name: QueueJobs.PkiSyncSyncCertificates;
        payload: TQueuePkiSyncSyncCertificatesByIdDTO;
      }
    | {
        name: QueueJobs.PkiSyncImportCertificates;
        payload: TQueuePkiSyncImportCertificatesByIdDTO;
      }
    | {
        name: QueueJobs.PkiSyncRemoveCertificates;
        payload: TQueuePkiSyncRemoveCertificatesByIdDTO;
      }
    | {
        name: QueueJobs.PkiSyncLinkMatchingCertificates;
        payload: TQueuePkiSyncLinkMatchingCertificatesDTO;
      }
    | {
        name: QueueJobs.PkiSyncReconcileFilters;
        payload: TQueuePkiSyncReconcileFiltersDTO;
      };
  [QueueName.PkiSyncHealthCheck]: {
    name: QueueJobs.PkiSyncRunHealthCheck;
    payload: TQueuePkiSyncRunHealthCheckByIdDTO;
  };
  [QueueName.ProjectV3Migration]: {
    name: QueueJobs.ProjectV3Migration;
    payload: { projectId: string };
  };
  [QueueName.FolderTreeCheckpoint]: {
    name: QueueJobs.CreateFolderTreeCheckpoint;
    payload: {
      envId: string;
    };
  };
  [QueueName.ImportSecretsFromExternalSource]: {
    name: QueueJobs.ImportSecretsFromExternalSource;
    payload: {
      orgId: string;
      actorId: string;
      actorEmail: string;
      importType: ExternalPlatforms;
      data: {
        iv: string;
        tag: string;
        ciphertext: string;
        algorithm: SecretEncryptionAlgo;
        encoding: SecretKeyEncoding;
      };
    };
  };
  [QueueName.VaultImportSideEffects]: {
    name: QueueJobs.VaultImportSideEffects;
    payload: TVaultImportSideEffectsJobPayload;
  };
  [QueueName.AppConnectionSecretSync]:
    | {
        name: QueueJobs.SecretSyncSyncSecrets;
        payload: TQueueSecretSyncSyncSecretsByIdDTO;
      }
    | {
        name: QueueJobs.SecretSyncImportSecrets;
        payload: TQueueSecretSyncImportSecretsByIdDTO;
      }
    | {
        name: QueueJobs.SecretSyncRemoveSecrets;
        payload: TQueueSecretSyncRemoveSecretsByIdDTO;
      }
    | {
        name: QueueJobs.SecretSyncSendActionFailedNotifications;
        payload: TQueueSendSecretSyncActionFailedNotificationsDTO;
      }
    | {
        name: QueueJobs.DailySecretSyncRetry;
        payload: undefined;
      };
  [QueueName.SecretRotationV2]:
    | {
        name: QueueJobs.SecretRotationV2QueueRotations;
        payload: undefined;
      }
    | {
        name: QueueJobs.SecretRotationV2SendNotification;
        payload: TSecretRotationSendNotificationJobPayload;
      };
  [QueueName.SecretRotationV2RotateSecrets]: {
    name: QueueJobs.SecretRotationV2RotateSecrets;
    payload: TSecretRotationRotateSecretsJobPayload;
  };
  [QueueName.PamCredentialRotation]: {
    name: QueueJobs.PamCredentialRotationQueueRotations;
    payload: undefined;
  };
  [QueueName.PamCredentialRotationRotate]: {
    name: QueueJobs.PamCredentialRotationRotate;
    payload: { accountId: string };
  };
  [QueueName.PamHeartbeat]: {
    name: QueueJobs.PamHeartbeatQueueChecks;
    payload: undefined;
  };
  [QueueName.PamHeartbeatCheck]: {
    name: QueueJobs.PamHeartbeatCheck;
    payload: { accountId: string };
  };
  [QueueName.InvalidateCache]: {
    name: QueueJobs.InvalidateCache;
    payload: {
      data: {
        type: CacheType;
      };
    };
  };
  [QueueName.SecretScanningV2]:
    | {
        name: QueueJobs.SecretScanningV2FullScan;
        payload: TQueueSecretScanningDataSourceFullScan;
      }
    | {
        name: QueueJobs.SecretScanningV2DiffScan;
        payload: TQueueSecretScanningResourceDiffScan;
      }
    | {
        name: QueueJobs.SecretScanningV2SendNotification;
        payload: TQueueSecretScanningSendNotification;
      }
    | {
        name: QueueJobs.SecretScanningV2DataSourceEvent;
        payload: TQueueSecretScanningDataSourceEvent;
      };
  [QueueName.SecretScanningV2FullScan]: {
    name: QueueJobs.SecretScanningV2FullScan;
    payload: TQueueSecretScanningDataSourceFullScan;
  };
  [QueueName.SecretScanningV2RealtimeScan]: {
    name: QueueJobs.SecretScanningV2DiffScan;
    payload: TQueueSecretScanningResourceDiffScanPayload;
  };
  [QueueName.CaLifecycle]: {
    name: QueueJobs.CaOrderCertificateForSubscriber;
    payload: {
      subscriberId: string;
      caType: CaType;
    };
  };
  [QueueName.CertificateIssuance]: {
    name: QueueJobs.CaIssueCertificateFromProfile;
    payload: {
      certificateId: string;
      profileId: string;
      caId: string;
      commonName?: string;
      altNames?: Array<{ type: string; value: string }>;
      ttl: string;
      signatureAlgorithm: string;
      keyAlgorithm: string;
      keyUsages?: string[];
      extendedKeyUsages?: string[];
      isRenewal?: boolean;
      originalCertificateId?: string;
      certificateRequestId?: string;
      csr?: string;
      organization?: string;
      organizationalUnit?: string;
      country?: string;
      state?: string;
      locality?: string;
      basicConstraints?: { isCA: boolean; pathLength?: number | null } | null;
    };
  };
  [QueueName.PkiSubscriber]: {
    name: QueueJobs.PkiSubscriberDailyAutoRenewal;
    payload: undefined;
  };
  [QueueName.UserNotification]: {
    name: QueueJobs.UserNotification;
    payload: { notifications: TCreateUserNotificationDTO[] };
  };
  [QueueName.AlertDispatch]: {
    name: QueueJobs.AlertDispatch;
    payload: { alertId: string; scheduledAt: string };
  };
  [QueueName.EventOutboxFlush]: {
    name: QueueJobs.EventOutboxFlush;
    payload: { consumer: string };
  };
  [QueueName.AuditReportGeneration]: {
    name: QueueJobs.GenerateAuditReport;
    payload: { auditReportId: string };
  };
  [QueueName.PamSessionExpiration]: {
    name: QueueJobs.PamSessionExpiration;
    payload: { sessionId: string };
  };
  [QueueName.PamDiscoveryScan]: {
    name: QueueJobs.PamDiscoverySourceScan;
    payload: { sourceId: string; triggeredBy: string };
  };
  [QueueName.PkiAcmeChallengeValidation]: {
    name: QueueJobs.PkiAcmeChallengeValidation;
    payload: { challengeId: string };
  };
  [QueueName.PkiDiscoveryScan]:
    | {
        name: QueueJobs.PkiDiscoveryRunScan;
        payload: { discoveryId: string };
      }
    | {
        name: QueueJobs.PkiDiscoveryScheduledScan;
        payload: undefined;
      };
  [QueueName.PkiDiscoveryRescan]: {
    name: QueueJobs.PkiDiscoveryRescanInstallation;
    payload: { installationId: string };
  };
  [QueueName.AppConnectionCredentialRotation]:
    | {
        name: QueueJobs.AppConnectionCredentialRotationQueueRotations;
        payload: undefined;
      }
    | {
        name: QueueJobs.AppConnectionCredentialRotationSendNotification;
        payload: TAppConnectionCredentialRotationSendNotificationJobPayload;
      };
  [QueueName.AppConnectionCredentialRotationRotate]: {
    name: QueueJobs.AppConnectionCredentialRotationRotate;
    payload: TAppConnectionCredentialRotationRotateJobPayload;
  };
  [QueueName.AuditLogClickHouseBatch]: {
    name: QueueJobs.AuditLogClickHouseBatch;
    payload: undefined;
  };
  [QueueName.AuditLogStreamOutbox]: {
    name: QueueJobs.AuditLogStreamFlush;
    payload: TAuditLogStreamFlushJobData;
  };
  [QueueName.AuditLogCollapsedFlush]: {
    name: QueueJobs.AuditLogCollapsedFlush;
    payload: TAuditLogCollapsedFlushJobData;
  };
  [QueueName.CaAutoRenewal]:
    | {
        name: QueueJobs.CaDailyAutoRenewal;
        payload: undefined;
      }
    | {
        name: QueueJobs.CaVenafiInstall;
        payload: { caId: string; maxPathLength?: number };
      }
    | {
        name: QueueJobs.CaAdcsInstall;
        payload: { caId: string; maxPathLength?: number };
      }
    | {
        name: QueueJobs.CaNativeAdcsInstall;
        payload: { caId: string; maxPathLength?: number };
      };
  [QueueName.ProjectHardDelete]: {
    name: QueueJobs.ProjectHardDelete;
    payload: { projectId: string };
  };
  [QueueName.ProjectEnvHardDelete]: {
    name: QueueJobs.ProjectEnvHardDelete;
    payload: { envId: string; projectId: string };
  };
  [QueueName.SignerAutoRenewal]: {
    name: QueueJobs.SignerDailyAutoRenewal;
    payload: undefined;
  };
  [QueueName.SecretBlindIndexMigration]: {
    name: QueueJobs.SecretBlindIndexMigration;
    // `scope` is absent on jobs queued before the org-wide walk shipped.
    payload: { scope: "org"; orgId: string } | { scope: "project"; projectId: string } | { projectId: string };
  };
  [QueueName.UsageEvent]: {
    name: QueueJobs.UsageEvent;
    payload: { orgId: string; dimensionKey: string };
  };
  [QueueName.IntegrationDeprecationNotice]: {
    name: QueueJobs.SendIntegrationDeprecationNotice;
    // period is a YYYY-MM stamp computed once by the cron tick so every retry of the same fire is deduped alike
    payload: { orgId: string; period: string };
  };
  [QueueName.LegacyPkiDeprecationNotice]: {
    name: QueueJobs.SendLegacyPkiDeprecationNotice;
    // period is a YYYY-MM stamp computed once by the cron tick so every retry of the same fire is deduped alike
    payload: { orgId: string; period: string };
  };
};

const SECRET_SCANNING_QUEUES = [
  QueueName.SecretScanningV2,
  QueueName.SecretScanningV2FullScan,
  QueueName.SecretScanningV2RealtimeScan,
  QueueName.SecretFullRepoScan,
  QueueName.SecretPushEventScan
];

const isQueueEnabled = (name: QueueName) => {
  const appCfg = getConfig();
  return SECRET_SCANNING_QUEUES.includes(name)
    ? appCfg.isSecretScanningRunModeEnabled
    : appCfg.isGeneralWorkerRunModeEnabled;
};

export type TQueueServiceFactory = {
  start: <T extends QueueName>(
    name: T,
    jobFn: (
      job: Job<TQueueJobTypes[T]["payload"], void, TQueueJobTypes[T]["name"]>,
      token?: string,
      signal?: AbortSignal
    ) => Promise<void>,
    queueSettings?: Omit<QueueOptions, "connection"> &
      Pick<WorkerOptions, "concurrency" | "limiter" | "maxStalledCount">
  ) => void;
  listen: <
    T extends QueueName,
    U extends keyof WorkerListener<TQueueJobTypes[T]["payload"], void, TQueueJobTypes[T]["name"]>
  >(
    name: T,
    event: U,
    listener: WorkerListener<TQueueJobTypes[T]["payload"], void, TQueueJobTypes[T]["name"]>[U]
  ) => void;
  queue: <T extends QueueName>(
    name: T,
    job: TQueueJobTypes[T]["name"],
    data: TQueueJobTypes[T]["payload"],
    opts: TQueueOptions
  ) => Promise<void>;
  shutdown: () => Promise<void>;
  // @deprecated Use removeJobScheduler instead.
  stopRepeatableJob: <T extends QueueName>(
    name: T,
    job: TQueueJobTypes[T]["name"],
    repeatOpt: RepeatOptions,
    jobId?: string
  ) => Promise<boolean | undefined>;
  // @deprecated Use stopJobById for delayed jobs. Use removeJobScheduler for schedulers.
  stopRepeatableJobByJobId: <T extends QueueName>(name: T, jobId: string) => Promise<boolean>;
  // @deprecated Use removeJobScheduler instead.
  stopRepeatableJobByKey: <T extends QueueName>(name: T, repeatJobKey: string) => Promise<boolean>;
  clearQueue: (name: QueueName) => Promise<void>;
  stopJobById: <T extends QueueName>(name: T, jobId: string) => Promise<void | undefined>;
  cancelActiveJob: <T extends QueueName>(name: T, jobId: string, reason?: string) => boolean;
  // @deprecated Use getJobSchedulers instead.
  getRepeatableJobs: (
    name: QueueName,
    startOffset?: number,
    endOffset?: number
  ) => Promise<{ key: string; name: string; id?: string | null }[]>;
  getDelayedJobs: (
    name: QueueName,
    startOffset?: number,
    endOffset?: number
  ) => Promise<{ delay: number; timestamp: number; repeatJobKey?: string; data?: unknown }[]>;
  upsertJobScheduler: <T extends QueueName>(
    name: T,
    schedulerId: string,
    repeatConfig: { pattern?: string; every?: number },
    jobTemplate?: {
      name?: TQueueJobTypes[T]["name"];
      data?: TQueueJobTypes[T]["payload"];
      opts?: {
        removeOnComplete?: boolean | { count: number };
        removeOnFail?: boolean | { count: number };
        delay?: number;
      };
    }
  ) => Promise<void>;
  removeJobScheduler: <T extends QueueName>(name: T, schedulerId: string) => Promise<void>;
  getJobSchedulers: (name: QueueName, start?: number, end?: number) => Promise<JobSchedulerJson[]>;
  getJob: <T extends QueueName>(
    name: T,
    jobId: string
  ) => Promise<Job<TQueueJobTypes[T]["payload"], void, string> | undefined>;
};

export const queueServiceFactory = (redisCfg: TRedisConfigKeys): TQueueServiceFactory => {
  const isClusterMode = Boolean(redisCfg?.REDIS_CLUSTER_HOSTS);
  const connection = buildRedisFromConfig(redisCfg, "queue");
  const queueContainer: Partial<Record<QueueName, Queue<TQueueJobTypes[QueueName]["payload"], void, string>>> = {};

  const workerContainer: Partial<
    Record<QueueName, Worker<TQueueJobTypes[QueueName]["payload"], void, TQueueJobTypes[QueueName]["name"]>>
  > = {};

  // One logger for every worker, so an outage hitting all of them still logs one line per error per window.
  const workerErrorLogger = createThrottledRedisErrorLogger("queue-workers", "Queue worker error");

  // Observable gauge for queue depth. The SDK invokes the callback on each export (every 30s for OTLP
  // push, on each scrape for Prometheus). Iterates only initialized queues in queueContainer; one
  // snapshot covers all ~30 named queues. Failures are swallowed because metrics must never crash the app.
  const QUEUE_DEPTH_STATES = ["waiting", "active", "delayed", "failed"] as const;
  const queueDepthGauge = resolveCoreMeter().createObservableGauge("infisical.queue.depth", {
    description: "Number of jobs in each queue state (waiting, active, delayed, failed)",
    unit: "{job}"
  });

  queueDepthGauge.addCallback(async (observableResult) => {
    if (!getConfig().OTEL_TELEMETRY_COLLECTION_ENABLED) return;
    await Promise.allSettled(
      Object.entries(queueContainer).map(async ([name, q]) => {
        if (!q) return;
        try {
          const counts = await q.getJobCounts(...QUEUE_DEPTH_STATES);
          Object.entries(counts).forEach(([state, count]) => {
            if (typeof count !== "number") return;
            observableResult.observe(count, { "queue.name": name, "queue.state": state });
          });
        } catch (err) {
          logger.warn({ err, queue: name }, `queue.depth gauge: getJobCounts failed [queue=${name}]`);
        }
      })
    );
  });

  // Remove orphaned job schedulers left in Redis by deleted queues.
  // Queues migrated to the cronJob system (cron-job.ts) are listed here so their
  // BullMQ schedulers and pending jobs are cleaned up on first boot of the new image.
  //
  // Gated to general-workers because every name below belongs to that fleet, and obliterate() is
  // called with force, which deletes active jobs too. Reaping is the consuming pod's job: an API
  // pod has no worker on these queues and must not clear work another pod is running.
  void (async () => {
    if (!getConfig().isGeneralWorkerRunModeEnabled) return;

    const staleQueueNames = [
      "queue-internal-recovery",
      "queue-internal-reconciliation",
      "secret-rotation",
      // Legacy per-log audit queue, replaced by the unified Redis ingest stream. The compatibility
      // shim that re-routed its jobs has been removed; obliterate any residue left in Redis.
      "audit-log",
      // Queues replaced by cronJobFactory (src/lib/cron/cron-job.ts)
      "daily-resource-cleanup",
      "frequent-resource-cleanup",
      "daily-reminders",
      "secret-reminder-migration",
      "health-alert",
      "certificate-cleanup",
      "pki-sync-cleanup",
      "pam-account-rotation",
      "pam-session-ai-summary",
      "daily-pki-alert-v2-processing",
      "daily-expiring-pki-item-alert",
      "telemtry-self-hosted-stats", // note: typo from original enum value
      "telemetry-aggregated-events",
      "certificate-v3-auto-renewal",
      "ca-crl-rotation",
      "digicert-order-polling"
    ];
    await Promise.allSettled(
      staleQueueNames.map(async (name) => {
        const staleQueue = new Queue(name, {
          prefix: isClusterMode ? `{${name}}` : undefined,
          connection
        });
        await staleQueue.obliterate({ force: true });
        await staleQueue.close();
        logger.info({ queue: name }, "Cleaned up orphaned internal queue from Redis");
      })
    );

    // Remove stale BullMQ schedulers for queues migrated to cronJobFactory.
    // Without this, the old scheduler keeps firing via the active Worker, causing double execution.
    const staleSchedulersInActiveQueues: Array<{ queueName: string; schedulerId: string }> = [
      { queueName: "pki-subscriber", schedulerId: `${JOB_SCHEDULER_PREFIX}:pki-subscriber` },
      { queueName: "pki-discovery-scan", schedulerId: `${JOB_SCHEDULER_PREFIX}:pki-discovery-scheduled-scan` },
      { queueName: "secret-rotation-v2", schedulerId: `${JOB_SCHEDULER_PREFIX}:secret-rotation-v2-cron` },
      {
        queueName: "app-connection-credential-rotation",
        schedulerId: `${JOB_SCHEDULER_PREFIX}:app-connection-credential-rotation-cron`
      },
      { queueName: "app-connection-secret-sync", schedulerId: `${JOB_SCHEDULER_PREFIX}:daily-secret-sync-retry-job` },
      { queueName: "ca-auto-renewal", schedulerId: `${JOB_SCHEDULER_PREFIX}:ca-daily-auto-renewal` }
    ];
    await Promise.allSettled(
      staleSchedulersInActiveQueues.map(async ({ queueName, schedulerId }) => {
        const q = new Queue(queueName, {
          prefix: isClusterMode ? `{${queueName}}` : undefined,
          connection
        });
        try {
          await q.removeJobScheduler(schedulerId);
          logger.info({ queue: queueName, schedulerId }, "Removed orphaned job scheduler from active queue");
        } catch (err) {
          logger.warn(
            { err, queue: queueName, schedulerId },
            "Failed to remove orphaned job scheduler from active queue"
          );
        } finally {
          await q.close();
        }
      })
    );
  })();

  const start: TQueueServiceFactory["start"] = (name, jobFn, queueSettings) => {
    if (queueContainer[name]) {
      throw new Error(`${name} queue is already initialized`);
    }

    const fipsSettings = crypto.isFipsModeEnabled() ? { settings: { repeatKeyHashAlgorithm: "sha256" as const } } : {};

    // The Queue (producer) is created regardless of run mode — only the Worker (consumer) is
    // gated below. A pod that doesn't consume a queue must still be able to enqueue onto it, or
    // splitting the fleet by run mode silently drops every job destined for another pod's worker.
    queueContainer[name] = new Queue(name as string, {
      prefix: isClusterMode ? `{${name}}` : undefined,
      ...queueSettings,
      ...fipsSettings,
      connection
    });

    if (!isQueueEnabled(name)) {
      return;
    }

    const worker = new Worker(name, jobFn, {
      prefix: isClusterMode ? `{${name}}` : undefined,
      ...fipsSettings,
      ...queueSettings,
      // Enable BullMQ's built-in per-minute completion/failure tracking in Redis. Survives pod restarts
      // (OTel cumulative counters reset), useful as a fallback when Prometheus retention is short.
      metrics: { maxDataPoints: MetricsTime.ONE_WEEK * 2 },
      connection
    });

    // Cross-cutting metric emission for every queue. Single registration covers all ~30 named queues
    // since BullMQ Worker is an EventEmitter and we attach the listeners here.
    worker.on("completed", (job) => {
      const baseAttrs = { "queue.name": name, "job.name": job.name } as Record<string, string>;
      queueJobCounter.add(1, { ...baseAttrs, outcome: "completed" });

      logger.debug({ queue: name, job: job.name, jobId: job.id }, "Queue job completed");

      if (typeof job.processedOn === "number" && typeof job.timestamp === "number") {
        const durationMs = Date.now() - job.processedOn;
        queueJobDurationHistogram.record(durationMs / 1000, { ...baseAttrs, outcome: "completed" });

        // Wait = (worker pickup) - (enqueue time + intentional delay). Subtracting opts.delay is what
        // makes this measure queue contention, not scheduling
        const intentionalDelayMs = job.opts.delay ?? 0;
        const waitMs = Math.max(0, job.processedOn - job.timestamp - intentionalDelayMs);
        queueJobWaitHistogram.record(waitMs / 1000, baseAttrs);
      }
    });

    worker.on("failed", (job, err) => {
      const baseAttrs = {
        "queue.name": name,
        "job.name": job?.name ?? "unknown"
      } as Record<string, string>;
      queueJobCounter.add(1, { ...baseAttrs, outcome: "failed" });

      // Skip duration on framework-level failures (job hydration failed before start). Emitting 0
      // would pollute the histogram with phantom zero-duration points.
      if (job && typeof job.processedOn === "number") {
        const durationMs = Date.now() - job.processedOn;
        queueJobDurationHistogram.record(durationMs / 1000, { ...baseAttrs, outcome: "failed" });
      }

      const errorType = classifyError(err);
      const attemptsExhausted = !!job && job.attemptsMade >= (job.opts.attempts || 1);
      queueJobFailureCounter.add(1, {
        ...baseAttrs,
        "error.type": errorType,
        "attempts.exhausted": attemptsExhausted ? "true" : "false"
      });

      logger.error(
        err,
        `Queue job failed [queue=${name}] [job=${job?.name ?? "unknown"}] [jobId=${job?.id ?? "unknown"}] [attempt=${
          job?.attemptsMade ?? 0
        }] [attemptsExhausted=${attemptsExhausted}]`
      );
    });

    // Stalling means the worker stopped heartbeating and BullMQ handed the job to someone else, so
    // it is both rare and always worth seeing. It stays at warn rather than debug for that reason.
    worker.on("stalled", (jobId) => {
      queueStalledCounter.add(1, { "queue.name": name });
      logger.warn({ queue: name, jobId }, `Queue job stalled and was requeued [queue=${name}] [jobId=${jobId}]`);
    });

    worker.on("active", (job) => {
      logger.debug({ queue: name, job: job.name, jobId: job.id }, "Queue job picked up by a worker");
    });

    // The worker's own blocking connection is a duplicate with no listeners, and BullMQ only reports its
    // errors (and run-loop errors) through this event.
    worker.on("error", (err) => workerErrorLogger.log(err, undefined, { queue: name }));

    workerContainer[name] = worker;
  };

  const listen: TQueueServiceFactory["listen"] = (name, event, listener) => {
    if (!isQueueEnabled(name)) {
      return;
    }

    const worker = workerContainer[name];
    worker?.on(event, listener);
  };

  const queue: TQueueServiceFactory["queue"] = async (name, job, data, opts) => {
    const q = queueContainer[name];

    const { jobId, repeat } = opts;
    const finalOptions = {
      removeOnFail: true,
      removeOnComplete: true,
      ...opts,
      repeat: repeat ? { ...repeat, utc: true } : undefined,
      jobId
    };

    const addedJob = await q?.add(job, data, finalOptions);

    // Only when the job really exists. `q` is undefined only where QUEUE_WORKERS_ENABLED is false,
    // and a pod in that mode neither produces nor consumes, so a no-op there is expected rather
    // than a dropped job worth reporting.
    if (addedJob) logger.debug({ queue: name, job, jobId: addedJob.id }, "Queue job enqueued");
  };

  const stopRepeatableJob: TQueueServiceFactory["stopRepeatableJob"] = async (name, job, repeatOpt, jobId) => {
    const q = queueContainer[name];
    if (q) {
      return q.removeRepeatable(job, repeatOpt, jobId);
    }
  };

  const getRepeatableJobs: TQueueServiceFactory["getRepeatableJobs"] = (name, startOffset, endOffset) => {
    const q = queueContainer[name];
    if (!q) throw new Error(`Queue '${name}' not initialized`);

    return q.getRepeatableJobs(startOffset, endOffset);
  };

  const getDelayedJobs: TQueueServiceFactory["getDelayedJobs"] = (name, startOffset, endOffset) => {
    const q = queueContainer[name];
    if (!q) throw new Error(`Queue '${name}' not initialized`);

    return q.getDelayed(startOffset, endOffset);
  };

  const stopRepeatableJobByJobId: TQueueServiceFactory["stopRepeatableJobByJobId"] = async (name, jobId) => {
    const q = queueContainer[name];
    if (!q) {
      return true;
    }

    const job = await q.getJob(jobId);
    if (!job?.repeatJobKey) {
      return true;
    }
    await job.remove();
    return q.removeRepeatableByKey(job.repeatJobKey);
  };

  const stopRepeatableJobByKey: TQueueServiceFactory["stopRepeatableJobByKey"] = async (name, repeatJobKey) => {
    const q = queueContainer[name];
    if (!q) {
      return true;
    }
    return q?.removeRepeatableByKey(repeatJobKey);
  };

  const stopJobById: TQueueServiceFactory["stopJobById"] = async (name, jobId) => {
    const q = queueContainer[name];
    const job = await q?.getJob(jobId);

    return job?.remove().catch(() => undefined);
  };

  const cancelActiveJob: TQueueServiceFactory["cancelActiveJob"] = (name, jobId, reason) => {
    const w = workerContainer[name];
    if (!w) return false;
    return w.cancelJob(jobId, reason);
  };

  const clearQueue: TQueueServiceFactory["clearQueue"] = async (name) => {
    const q = queueContainer[name];
    await q?.drain();
  };

  const shutdown: TQueueServiceFactory["shutdown"] = async () => {
    await Promise.all(Object.values(workerContainer).map((worker) => worker.close()));
  };

  const upsertJobScheduler: TQueueServiceFactory["upsertJobScheduler"] = async (
    name,
    schedulerId,
    repeatConfig,
    jobTemplate
  ) => {
    const q = queueContainer[name];
    if (!q) throw new Error(`Queue '${name}' not initialized`);

    // Remove legacy repeatable jobs that don't use the job scheduler prefix.
    // This prevents duplicate execution after migrating from queue.add({repeat}) to upsertJobScheduler.
    try {
      const repeatableJobs = await q.getRepeatableJobs();
      const legacyJobs = repeatableJobs.filter((job) => !job.key.startsWith(JOB_SCHEDULER_PREFIX));
      await Promise.all(
        legacyJobs.map((job) =>
          q.removeRepeatableByKey(job.key).then(() => {
            logger.info({ queue: name, key: job.key }, "Removed legacy repeatable job");
          })
        )
      );
    } catch (err) {
      logger.error(err, `Failed to clean up legacy repeatable jobs for queue '${name}'`);
    }

    // Remove orphaned delayed jobs left behind by legacy repeatable schedules.
    // removeRepeatableByKey cleans up the ZSET entry and its delayed job, but workers may
    // have already processed the old delayed job and re-created the next iteration before
    // the ZSET entry was removed. Runs before and after upsert to handle the race window.
    const $removeLegacyDelayedJobs = async () => {
      const delayedJobs = await q.getDelayed();
      const legacyDelayedJobs = delayedJobs.filter(
        (job) => job?.id?.startsWith("repeat:") && !job.id.includes(JOB_SCHEDULER_PREFIX)
      );
      await Promise.all(
        legacyDelayedJobs.map((job) =>
          job
            .remove()
            .then(() => {
              logger.info({ queue: name, jobId: job.id }, "Removed orphaned legacy delayed job");
            })
            .catch((removeErr: unknown) => {
              logger.warn(
                { queue: name, jobId: job.id, error: removeErr },
                "Failed to remove orphaned legacy delayed job"
              );
            })
        )
      );
    };

    try {
      await $removeLegacyDelayedJobs();
    } catch (err) {
      logger.error(err, `Failed to clean up orphaned legacy delayed jobs for queue '${name}'`);
    }

    await q.upsertJobScheduler(
      schedulerId,
      { ...repeatConfig, utc: true },
      {
        name: jobTemplate?.name ?? schedulerId,
        data: jobTemplate?.data,
        opts: {
          removeOnComplete: true,
          removeOnFail: true,
          ...jobTemplate?.opts
        }
      }
    );

    try {
      await $removeLegacyDelayedJobs();
    } catch (err) {
      logger.error(err, `Failed second-pass cleanup of orphaned legacy delayed jobs for queue '${name}'`);
    }
  };

  const removeJobScheduler: TQueueServiceFactory["removeJobScheduler"] = async (name, schedulerId) => {
    const q = queueContainer[name];
    if (!q) throw new Error(`Queue '${name}' not initialized`);

    await q.removeJobScheduler(schedulerId);
  };

  const getJobSchedulers: TQueueServiceFactory["getJobSchedulers"] = async (name, startOffset, endOffset) => {
    const q = queueContainer[name];
    if (!q) throw new Error(`Queue '${name}' not initialized`);

    return q.getJobSchedulers(startOffset, endOffset);
  };

  const getJob: TQueueServiceFactory["getJob"] = async (name, jobId) => {
    const q = queueContainer[name];
    if (!q) return undefined;
    const job = await q.getJob(jobId);
    return job ?? undefined;
  };

  return {
    start,
    listen,
    queue,
    shutdown,
    stopRepeatableJob,
    stopRepeatableJobByJobId,
    stopRepeatableJobByKey,
    clearQueue,
    stopJobById,
    cancelActiveJob,
    getRepeatableJobs,
    getDelayedJobs,
    upsertJobScheduler,
    removeJobScheduler,
    getJobSchedulers,
    getJob
  };
};