Infisical is the open-source platform for securing secrets, certificates, access, and AI Agents.
TypeScript acmeagentsai-agentscertificate-managementdevsecopsenvironment-variableskubernetesopen-sourcepampkipostgresprivate-caprivileged-access-managementsecret-managementsecret-managersecretssecrets-managementsecuritysecurity-toolsvault
create new queues for secret-scanning v2
Matheus Nogueira16e3c487cf59
diff --git a/backend/src/ee/services/secret-scanning-v2/secret-scanning-v2-queue.ts b/backend/src/ee/services/secret-scanning-v2/secret-scanning-v2-queue.tsindex 3d1ae084f8..3f11608725 100644--- a/backend/src/ee/services/secret-scanning-v2/secret-scanning-v2-queue.ts+++ b/backend/src/ee/services/secret-scanning-v2/secret-scanning-v2-queue.ts@@ -41,6 +41,7 @@ import {TFindingsPayload,TQueueSecretScanningDataSourceFullScan,TQueueSecretScanningResourceDiffScan,+ TQueueSecretScanningResourceDiffScanPayload,TQueueSecretScanningSendNotification,TSecretScanningDataSourceWithConnection,TSecretScanningFinding@@ -144,7 +145,7 @@ export const secretScanningV2QueueServiceFactory = ({for (const scan of scans) {// eslint-disable-next-line no-await-in-loopawait queueService.queue(- QueueName.SecretScanningV2,+ QueueName.SecretScanningV2FullScan,QueueJobs.SecretScanningV2FullScan,{scanId: scan.id,@@ -409,7 +410,7 @@ export const secretScanningV2QueueServiceFactory = ({payload,dataSourceId,dataSourceType- }: Pick<TQueueSecretScanningResourceDiffScan, "payload" | "dataSourceId" | "dataSourceType">) => {+ }: Omit<TQueueSecretScanningResourceDiffScanPayload, "scanId" | "resourceId">) => {const factory = SECRET_SCANNING_FACTORY_MAP[dataSourceType as SecretScanningDataSource]({kmsService,appConnectionDAL@@ -445,7 +446,7 @@ export const secretScanningV2QueueServiceFactory = ({});await queueService.queue(- QueueName.SecretScanningV2,+ QueueName.SecretScanningV2DiffScan,QueueJobs.SecretScanningV2DiffScan,{payload,@@ -854,7 +855,25 @@ export const secretScanningV2QueueServiceFactory = ({}});+ queueService.start(+ QueueName.SecretScanningV2FullScan,+ async (job) => {+ await handleFullScan(job as Parameters<typeof handleFullScan>[0]);+ },+ { concurrency: 1 }+ );++ queueService.start(+ QueueName.SecretScanningV2DiffScan,+ async (job) => {+ await handleDiffScan(job as Parameters<typeof handleDiffScan>[0]);+ },+ { concurrency: 5 }+ );+queueService.start(QueueName.SecretScanningV2, async (job) => {+ // We are keeping this for now because once deployed, the queue might still have+ // full scan and diff scan messages in it and need to be processed.if (job.name === QueueJobs.SecretScanningV2FullScan) {await handleFullScan(job as Parameters<typeof handleFullScan>[0]);} else if (job.name === QueueJobs.SecretScanningV2DiffScan) {diff --git a/backend/src/ee/services/secret-scanning-v2/secret-scanning-v2-types.ts b/backend/src/ee/services/secret-scanning-v2/secret-scanning-v2-types.tsindex 6c5d86d16e..d885b05f5f 100644--- a/backend/src/ee/services/secret-scanning-v2/secret-scanning-v2-types.ts+++ b/backend/src/ee/services/secret-scanning-v2/secret-scanning-v2-types.ts@@ -138,6 +138,14 @@ export type TQueueSecretScanningResourceDiffScan =| TQueueBitbucketResourceDiffScan| TQueueGitLabResourceDiffScan;+export type TQueueSecretScanningResourceDiffScanPayload = {+ payload: TQueueSecretScanningResourceDiffScan["payload"];+ dataSourceType: SecretScanningDataSource;+ dataSourceId: string;+ resourceId: string;+ scanId: string;+};+export type TQueueSecretScanningSendNotification = {dataSource: TSecretScanningDataSources;resourceName: string;diff --git a/backend/src/queue/queue-service.ts b/backend/src/queue/queue-service.tsindex c28461c63a..818b085b11 100644--- a/backend/src/queue/queue-service.ts+++ b/backend/src/queue/queue-service.ts@@ -23,6 +23,7 @@ import {import {TQueueSecretScanningDataSourceFullScan,TQueueSecretScanningResourceDiffScan,+ TQueueSecretScanningResourceDiffScanPayload,TQueueSecretScanningSendNotification} from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-types";import { getConfig } from "@app/lib/config/env";@@ -103,6 +104,8 @@ export enum QueueName {FolderTreeCheckpoint = "folder-tree-checkpoint",InvalidateCache = "invalidate-cache",SecretScanningV2 = "secret-scanning-v2",+ SecretScanningV2FullScan = "secret-scanning-v2-full-scan",+ SecretScanningV2DiffScan = "secret-scanning-v2-diff-scan",UserNotification = "user-notification",AlertDispatch = "alert-dispatch",AuditReportGeneration = "audit-report-generation",@@ -461,6 +464,14 @@ export type TQueueJobTypes = {name: QueueJobs.SecretScanningV2SendNotification;payload: TQueueSecretScanningSendNotification;};+ [QueueName.SecretScanningV2FullScan]: {+ name: QueueJobs.SecretScanningV2FullScan;+ payload: TQueueSecretScanningDataSourceFullScan;+ };+ [QueueName.SecretScanningV2DiffScan]: {+ name: QueueJobs.SecretScanningV2DiffScan;+ payload: TQueueSecretScanningResourceDiffScanPayload;+ };[QueueName.CaLifecycle]: {name: QueueJobs.CaOrderCertificateForSubscriber;payload: {@@ -597,6 +608,8 @@ export type TQueueJobTypes = {const SECRET_SCANNING_QUEUES = [QueueName.SecretScanningV2,+ QueueName.SecretScanningV2FullScan,+ QueueName.SecretScanningV2DiffScan,QueueName.SecretFullRepoScan,QueueName.SecretPushEventScan];