Skip to content

Infisical is the open-source platform for securing secrets, certificates, access, and AI Agents.

make retries be handled via bullmq

Matheus Nogueira3b234a26af2f
diff --git a/backend/src/ee/services/secret-scanning-v2/secret-scanning-v2-queue.ts b/backend/src/ee/services/secret-scanning-v2/secret-scanning-v2-queue.ts
index 2efc0c8848..e64947653a 100644
--- a/backend/src/ee/services/secret-scanning-v2/secret-scanning-v2-queue.ts
+++ b/backend/src/ee/services/secret-scanning-v2/secret-scanning-v2-queue.ts
@@ -12,11 +12,11 @@ import {
parseScanErrorMessage,
scanGitRepositoryAndGetFindings
} from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-fns";
-import { KeyStorePrefixes, TKeyStoreFactory } from "@app/keystore/keystore";
-import { getConfig, SECRET_SCANNING_SCAN_OVERHEAD } from "@app/lib/config/env";
+import { getConfig } from "@app/lib/config/env";
import { CronJobName, TCronJobFactory } from "@app/lib/cron/cron-job";
import { BadRequestError, InternalServerError } from "@app/lib/errors";
import { logger } from "@app/lib/logger";
+import { ms } from "@app/lib/ms";
import { QueueJobs, QueueName, TQueueServiceFactory } from "@app/queue";
import { TAppConnectionDALFactory } from "@app/services/app-connection/app-connection-dal";
import { decryptAppConnection } from "@app/services/app-connection/app-connection-fns";
@@ -57,7 +57,6 @@ type TSecretRotationV2QueueServiceFactoryDep = {
kmsService: Pick<TKmsServiceFactory, "createCipherPairWithDataKey">;
appConnectionDAL: Pick<TAppConnectionDALFactory, "updateById">;
auditLogService: Pick<TAuditLogServiceFactory, "createAuditLog">;
- keyStore: Pick<TKeyStoreFactory, "acquireLock" | "getItem">;
notificationService: Pick<TNotificationServiceFactory, "createUserNotifications">;
};
@@ -67,13 +66,14 @@ const STUCK_SCAN_REAP_BATCH_SIZE = 100;
const STUCK_SCAN_STATUS_MESSAGE =
"The scan did not complete and was cancelled. This usually means the resource is too large to scan.";
-// The lock has to outlive the worst case a scan can take: clone timeout plus scan timeout plus the
-// same measurement/bookkeeping overhead the stuck-scan boot validation budgets for. Anything
-// shorter can expire mid-scan and let a second scan of the same resource start alongside the first.
-const getFullScanLockTtlMs = () => {
- const appCfg = getConfig();
- return appCfg.SECRET_SCANNING_CLONE_TIMEOUT + appCfg.SECRET_SCANNING_SCAN_TIMEOUT + SECRET_SCANNING_SCAN_OVERHEAD;
-};
+// A full scan usually dies because the worker was OOM-killed on a large repository, which retrying
+// does resolve — the pod that picks it up next is not the one that ran out of memory. The two
+// budgets are separate in BullMQ and both are needed: `attempts` covers a run that throws, while a
+// killed process never throws at all and is recovered by the stalled checker under
+// `maxStalledCount`. The delay only has to outlast a restarting pod; nothing is held across it.
+const FULL_SCAN_ATTEMPTS = 3;
+const FULL_SCAN_MAX_STALLED_COUNT = 2;
+const FULL_SCAN_RETRY_DELAY = ms("1m");
export const secretScanningV2QueueServiceFactory = ({
queueService,
@@ -84,7 +84,6 @@ export const secretScanningV2QueueServiceFactory = ({
smtpService,
kmsService,
auditLogService,
- keyStore,
appConnectionDAL,
notificationService
}: TSecretRotationV2QueueServiceFactoryDep) => {
@@ -115,13 +114,6 @@ export const secretScanningV2QueueServiceFactory = ({
});
}
- for (const resource of filteredRawResources) {
- // eslint-disable-next-line no-await-in-loop
- if (await keyStore.getItem(KeyStorePrefixes.SecretScanningLock(dataSource.id, resource.externalId))) {
- throw new BadRequestError({ message: `A scan is already in progress for resource "${resource.name}"` });
- }
- }
-
await secretScanningV2DAL.resources.transaction(async (tx) => {
const resources = await secretScanningV2DAL.resources.upsert(
filteredRawResources.map((rawResource) => ({
@@ -132,8 +124,24 @@ export const secretScanningV2QueueServiceFactory = ({
tx
);
+ const inFlightScans = await secretScanningV2DAL.scans.find(
+ {
+ type: SecretScanningScanType.FullScan,
+ $in: {
+ resourceId: resources.map((resource) => resource.id),
+ status: [SecretScanningScanStatus.Queued, SecretScanningScanStatus.Scanning]
+ }
+ },
+ { tx }
+ );
+
+ const scannedResourceIds = new Set(inFlightScans.map((scan) => scan.resourceId));
+ const resourcesToScan = resources.filter((resource) => !scannedResourceIds.has(resource.id));
+
+ if (!resourcesToScan.length) return;
+
const scans = await secretScanningV2DAL.scans.insertMany(
- resources.map((resource) => ({
+ resourcesToScan.map((resource) => ({
resourceId: resource.id,
type: SecretScanningScanType.FullScan
})),
@@ -150,7 +158,12 @@ export const secretScanningV2QueueServiceFactory = ({
resourceId: scan.resourceId,
dataSourceId: dataSource.id
},
- { jobId: scan.id, removeOnFail: true }
+ {
+ jobId: scan.id,
+ removeOnFail: true,
+ attempts: FULL_SCAN_ATTEMPTS,
+ backoff: { type: "fixed", delay: FULL_SCAN_RETRY_DELAY }
+ }
);
}
});
@@ -191,18 +204,7 @@ export const secretScanningV2QueueServiceFactory = ({
if (!resource) throw new Error(`Resource with ID "${resourceId}" not found`);
- let lock: Awaited<ReturnType<typeof keyStore.acquireLock>> | undefined;
-
try {
- try {
- lock = await keyStore.acquireLock(
- [KeyStorePrefixes.SecretScanningLock(dataSource.id, resource.externalId)],
- getFullScanLockTtlMs()
- );
- } catch (e) {
- throw new Error("Failed to acquire scanning lock.");
- }
-
await secretScanningV2DAL.scans.update(
{ id: scanId },
{
@@ -400,7 +402,6 @@ export const secretScanningV2QueueServiceFactory = ({
throw error;
} finally {
await deleteTempFolder(tempFolder);
- await lock?.release();
}
};
@@ -858,7 +859,7 @@ export const secretScanningV2QueueServiceFactory = ({
async (job) => {
await handleFullScan(job as Parameters<typeof handleFullScan>[0]);
},
- { concurrency: 1 }
+ { concurrency: 1, maxStalledCount: FULL_SCAN_MAX_STALLED_COUNT }
);
queueService.start(
diff --git a/backend/src/keystore/keystore.ts b/backend/src/keystore/keystore.ts
index 3e893fac08..3576deb50a 100644
--- a/backend/src/keystore/keystore.ts
+++ b/backend/src/keystore/keystore.ts
@@ -69,8 +69,6 @@ export const KeyStorePrefixes = {
`ldap-directory-machines-${connectionId}-${limit}-${search}` as const,
SecretRotationLock: (rotationId: string) => `secret-rotation-v2-mutex-${rotationId}` as const,
PamAccountRotationLock: (accountId: string) => `pam-account-rotation-mutex-${accountId}` as const,
- SecretScanningLock: (dataSourceId: string, resourceExternalId: string) =>
- `secret-scanning-v2-mutex-${dataSourceId}-${resourceExternalId}` as const,
IdentityLockoutLock: (lockoutKey: string) => `identity-lockout-lock-${lockoutKey}` as const,
CaOrderCertificateForSubscriberLock: (subscriberId: string) =>
`ca-order-certificate-for-subscriber-lock-${subscriberId}` as const,
diff --git a/backend/src/queue/queue-service.ts b/backend/src/queue/queue-service.ts
index 0b2567f32b..ff248b59e9 100644
--- a/backend/src/queue/queue-service.ts
+++ b/backend/src/queue/queue-service.ts
@@ -652,7 +652,8 @@ export type TQueueServiceFactory = {
token?: string,
signal?: AbortSignal
) => Promise<void>,
- queueSettings?: Omit<QueueOptions, "connection"> & Pick<WorkerOptions, "concurrency" | "limiter">
+ queueSettings?: Omit<QueueOptions, "connection"> &
+ Pick<WorkerOptions, "concurrency" | "limiter" | "maxStalledCount">
) => void;
listen: <
T extends QueueName,
diff --git a/backend/src/server/routes/index.ts b/backend/src/server/routes/index.ts
index 8cec726cd8..9bcbb729ac 100644
--- a/backend/src/server/routes/index.ts
+++ b/backend/src/server/routes/index.ts
@@ -4057,7 +4057,6 @@ export const registerRoutes = async (
projectMembershipDAL,
smtpService,
kmsService,
- keyStore,
appConnectionDAL,
notificationService
});