Skip to content

Infisical is the open-source platform for securing secrets, certificates, access, and AI Agents.

create new queues for secret-scanning v2

Matheus Nogueira16e3c487cf59
diff --git a/backend/src/ee/services/secret-scanning-v2/secret-scanning-v2-queue.ts b/backend/src/ee/services/secret-scanning-v2/secret-scanning-v2-queue.ts
index 3d1ae084f8..3f11608725 100644
--- a/backend/src/ee/services/secret-scanning-v2/secret-scanning-v2-queue.ts
+++ b/backend/src/ee/services/secret-scanning-v2/secret-scanning-v2-queue.ts
@@ -41,6 +41,7 @@ import {
TFindingsPayload,
TQueueSecretScanningDataSourceFullScan,
TQueueSecretScanningResourceDiffScan,
+ TQueueSecretScanningResourceDiffScanPayload,
TQueueSecretScanningSendNotification,
TSecretScanningDataSourceWithConnection,
TSecretScanningFinding
@@ -144,7 +145,7 @@ export const secretScanningV2QueueServiceFactory = ({
for (const scan of scans) {
// eslint-disable-next-line no-await-in-loop
await queueService.queue(
- QueueName.SecretScanningV2,
+ QueueName.SecretScanningV2FullScan,
QueueJobs.SecretScanningV2FullScan,
{
scanId: scan.id,
@@ -409,7 +410,7 @@ export const secretScanningV2QueueServiceFactory = ({
payload,
dataSourceId,
dataSourceType
- }: Pick<TQueueSecretScanningResourceDiffScan, "payload" | "dataSourceId" | "dataSourceType">) => {
+ }: Omit<TQueueSecretScanningResourceDiffScanPayload, "scanId" | "resourceId">) => {
const factory = SECRET_SCANNING_FACTORY_MAP[dataSourceType as SecretScanningDataSource]({
kmsService,
appConnectionDAL
@@ -445,7 +446,7 @@ export const secretScanningV2QueueServiceFactory = ({
});
await queueService.queue(
- QueueName.SecretScanningV2,
+ QueueName.SecretScanningV2DiffScan,
QueueJobs.SecretScanningV2DiffScan,
{
payload,
@@ -854,7 +855,25 @@ export const secretScanningV2QueueServiceFactory = ({
}
});
+ queueService.start(
+ QueueName.SecretScanningV2FullScan,
+ async (job) => {
+ await handleFullScan(job as Parameters<typeof handleFullScan>[0]);
+ },
+ { concurrency: 1 }
+ );
+
+ queueService.start(
+ QueueName.SecretScanningV2DiffScan,
+ async (job) => {
+ await handleDiffScan(job as Parameters<typeof handleDiffScan>[0]);
+ },
+ { concurrency: 5 }
+ );
+
queueService.start(QueueName.SecretScanningV2, async (job) => {
+ // We are keeping this for now because once deployed, the queue might still have
+ // full scan and diff scan messages in it and need to be processed.
if (job.name === QueueJobs.SecretScanningV2FullScan) {
await handleFullScan(job as Parameters<typeof handleFullScan>[0]);
} else if (job.name === QueueJobs.SecretScanningV2DiffScan) {
diff --git a/backend/src/ee/services/secret-scanning-v2/secret-scanning-v2-types.ts b/backend/src/ee/services/secret-scanning-v2/secret-scanning-v2-types.ts
index 6c5d86d16e..d885b05f5f 100644
--- a/backend/src/ee/services/secret-scanning-v2/secret-scanning-v2-types.ts
+++ b/backend/src/ee/services/secret-scanning-v2/secret-scanning-v2-types.ts
@@ -138,6 +138,14 @@ export type TQueueSecretScanningResourceDiffScan =
| TQueueBitbucketResourceDiffScan
| TQueueGitLabResourceDiffScan;
+export type TQueueSecretScanningResourceDiffScanPayload = {
+ payload: TQueueSecretScanningResourceDiffScan["payload"];
+ dataSourceType: SecretScanningDataSource;
+ dataSourceId: string;
+ resourceId: string;
+ scanId: string;
+};
+
export type TQueueSecretScanningSendNotification = {
dataSource: TSecretScanningDataSources;
resourceName: string;
diff --git a/backend/src/queue/queue-service.ts b/backend/src/queue/queue-service.ts
index c28461c63a..818b085b11 100644
--- a/backend/src/queue/queue-service.ts
+++ b/backend/src/queue/queue-service.ts
@@ -23,6 +23,7 @@ import {
import {
TQueueSecretScanningDataSourceFullScan,
TQueueSecretScanningResourceDiffScan,
+ TQueueSecretScanningResourceDiffScanPayload,
TQueueSecretScanningSendNotification
} from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-types";
import { getConfig } from "@app/lib/config/env";
@@ -103,6 +104,8 @@ export enum QueueName {
FolderTreeCheckpoint = "folder-tree-checkpoint",
InvalidateCache = "invalidate-cache",
SecretScanningV2 = "secret-scanning-v2",
+ SecretScanningV2FullScan = "secret-scanning-v2-full-scan",
+ SecretScanningV2DiffScan = "secret-scanning-v2-diff-scan",
UserNotification = "user-notification",
AlertDispatch = "alert-dispatch",
AuditReportGeneration = "audit-report-generation",
@@ -461,6 +464,14 @@ export type TQueueJobTypes = {
name: QueueJobs.SecretScanningV2SendNotification;
payload: TQueueSecretScanningSendNotification;
};
+ [QueueName.SecretScanningV2FullScan]: {
+ name: QueueJobs.SecretScanningV2FullScan;
+ payload: TQueueSecretScanningDataSourceFullScan;
+ };
+ [QueueName.SecretScanningV2DiffScan]: {
+ name: QueueJobs.SecretScanningV2DiffScan;
+ payload: TQueueSecretScanningResourceDiffScanPayload;
+ };
[QueueName.CaLifecycle]: {
name: QueueJobs.CaOrderCertificateForSubscriber;
payload: {
@@ -597,6 +608,8 @@ export type TQueueJobTypes = {
const SECRET_SCANNING_QUEUES = [
QueueName.SecretScanningV2,
+ QueueName.SecretScanningV2FullScan,
+ QueueName.SecretScanningV2DiffScan,
QueueName.SecretFullRepoScan,
QueueName.SecretPushEventScan
];