Skip to content

Infisical is the open-source platform for securing secrets, certificates, access, and AI Agents.

cleanup

Matheus Nogueiraf0e99b14195a
diff --git a/backend/src/ee/services/secret-scanning-v2/secret-scanning-v2-fns.test.ts b/backend/src/ee/services/secret-scanning-v2/secret-scanning-v2-fns.test.ts
index 80fd50086f..fe9db9b273 100644
--- a/backend/src/ee/services/secret-scanning-v2/secret-scanning-v2-fns.test.ts
+++ b/backend/src/ee/services/secret-scanning-v2/secret-scanning-v2-fns.test.ts
@@ -211,7 +211,6 @@ describe("planCommitBatches", () => {
});
expect(resumed.resumed).toBe(true);
- expect(resumed.prefixChanged).toBe(false);
expect(resumed.batches).toEqual(first.batches.slice(1));
});
});
@@ -257,7 +256,6 @@ describe("planCommitBatches", () => {
resumeAfterCommitDigest: firstBatch.prefixDigest
});
- expect(resumed.prefixChanged).toBe(true);
expect(resumed.resumed).toBe(false);
// Every commit is back in the plan, the never-scanned one included.
@@ -281,7 +279,6 @@ describe("planCommitBatches", () => {
});
expect(plan.resumed).toBe(false);
- expect(plan.prefixChanged).toBe(false);
expect(plan.batches).toHaveLength(2);
});
});
diff --git a/backend/src/ee/services/secret-scanning-v2/secret-scanning-v2-fns.ts b/backend/src/ee/services/secret-scanning-v2/secret-scanning-v2-fns.ts
index deedf3a644..0c6e40f63f 100644
--- a/backend/src/ee/services/secret-scanning-v2/secret-scanning-v2-fns.ts
+++ b/backend/src/ee/services/secret-scanning-v2/secret-scanning-v2-fns.ts
@@ -200,8 +200,6 @@ export const planCommitBatches = async ({
totalCommits: number;
batches: TCommitBatch[];
resumed: boolean;
- /** Set when the resume point is still present but the history before it is not the history that was scanned. */
- prefixChanged: boolean;
}> => {
const boundaries: { index: number; commit: string; prefixDigest: string }[] = [];
let totalCommits = 0;
@@ -252,6 +250,16 @@ export const planCommitBatches = async ({
const prefixChanged = resumeIndex >= 0 && resumePrefixDigest !== resumeAfterCommitDigest;
const resumableIndex = prefixChanged ? -1 : resumeIndex;
+ // Two ways a resume point stops meaning anything: a rewritten history (force push, or a rebase
+ // landing between runs) takes the commit out of the repository entirely, or a newly reachable ref
+ // puts commits ahead of it that this scan has never looked at. Either way the repository is
+ // re-walked from the start rather than resumed past commits nobody scanned.
+ if (resumeAfterCommit && resumableIndex < 0) {
+ logger.warn(
+ `secretScanningV2: Full Scan cannot resume, restarting [repoPath=${repoPath}] [lastScannedCommit=${resumeAfterCommit}] [reason=${prefixChanged ? "history before the resume point changed" : "resume point is no longer in the repository"}]`
+ );
+ }
+
const batches = boundaries
.map(({ index, commit, prefixDigest: boundaryPrefixDigest }, position) => ({
index,
@@ -268,7 +276,7 @@ export const planCommitBatches = async ({
prefixDigest: boundaryPrefixDigest
}));
- return { totalCommits, batches, resumed: resumableIndex >= 0, prefixChanged };
+ return { totalCommits, batches, resumed: resumableIndex >= 0 };
};
export async function scanDirectory(
diff --git a/backend/src/ee/services/secret-scanning-v2/secret-scanning-v2-queue.ts b/backend/src/ee/services/secret-scanning-v2/secret-scanning-v2-queue.ts
index ba7e5b72f8..e1f6e1f450 100644
--- a/backend/src/ee/services/secret-scanning-v2/secret-scanning-v2-queue.ts
+++ b/backend/src/ee/services/secret-scanning-v2/secret-scanning-v2-queue.ts
@@ -124,6 +124,49 @@ export const secretScanningV2QueueServiceFactory = ({
if ((await keyStore.getItemPrimary(key)) === scanId) await keyStore.deleteItem(key);
};
+ /**
+ * Guarded so a retry cannot reopen a scan that already reached a terminal status. The work itself
+ * succeeded and only a side effect after it (a notification enqueue, an audit log) threw, so
+ * BullMQ hands the whole handler back; without this an already-Completed scan would be reset to
+ * `scanning` and, on the last attempt, closed out as Failed. Returns false when the scan was
+ * already closed out.
+ */
+ const markScanAsScanning = async ({ scanId, trackProgress }: { scanId: string; trackProgress?: boolean }) => {
+ const startedScans = await secretScanningV2DAL.scans.update(
+ {
+ id: scanId,
+ $in: { status: [SecretScanningScanStatus.Queued, SecretScanningScanStatus.Scanning] }
+ },
+ {
+ status: SecretScanningScanStatus.Scanning,
+ scanningStartedAt: new Date(),
+ ...(trackProgress ? { progressUpdatedAt: new Date() } : {})
+ }
+ );
+
+ return startedScans.length > 0;
+ };
+
+ /**
+ * `queued` is accepted alongside `scanning` because a failure before the status was set leaves the
+ * row `queued`. A scan the reaper has already failed and notified on is left exactly as it is,
+ * which is what the returned flag reports.
+ */
+ const markScanAsFailed = async ({ scanId, statusMessage }: { scanId: string; statusMessage: string }) => {
+ const failedScans = await secretScanningV2DAL.scans.update(
+ {
+ id: scanId,
+ $in: { status: [SecretScanningScanStatus.Queued, SecretScanningScanStatus.Scanning] }
+ },
+ {
+ status: SecretScanningScanStatus.Failed,
+ statusMessage
+ }
+ );
+
+ return failedScans.length > 0;
+ };
+
const queueDataSourceFullScan = async (
dataSource: TSecretScanningDataSourceWithConnection,
resourceExternalId?: string
@@ -251,39 +294,16 @@ export const secretScanningV2QueueServiceFactory = ({
holdsLease = await acquireFullScanLease(resourceId, scanId);
if (!holdsLease) {
- await secretScanningV2DAL.scans.update(
- {
- id: scanId,
- $in: { status: [SecretScanningScanStatus.Queued, SecretScanningScanStatus.Scanning] }
- },
- {
- status: SecretScanningScanStatus.Failed,
- statusMessage: DUPLICATE_FULL_SCAN_STATUS_MESSAGE
- }
- );
+ await markScanAsFailed({ scanId, statusMessage: DUPLICATE_FULL_SCAN_STATUS_MESSAGE });
logger.warn(`secretScanningV2Queue: Full Scan Skipped, resource is already being scanned ${logDetails}`);
return;
}
- // Guarded so a retry cannot reopen a scan that already reached a terminal status. The work
- // itself succeeded and only a side effect after it (a notification enqueue, an audit log)
- // threw, so BullMQ hands the whole handler back; without this an already-Completed scan would
- // be reset to `scanning` and, on the last attempt, closed out as Failed.
- const startedScans = await secretScanningV2DAL.scans.update(
- {
- id: scanId,
- $in: { status: [SecretScanningScanStatus.Queued, SecretScanningScanStatus.Scanning] }
- },
- {
- status: SecretScanningScanStatus.Scanning,
- scanningStartedAt: new Date(),
- progressUpdatedAt: new Date()
- }
- );
+ const started = await markScanAsScanning({ scanId, trackProgress: true });
- if (!startedScans.length) {
+ if (!started) {
logger.warn(
`secretScanningV2Queue: Full Scan skipped, scan was already closed out ${logDetails} [scanType=${SecretScanningScanType.FullScan}]`
);
@@ -400,16 +420,6 @@ export const secretScanningV2QueueServiceFactory = ({
resumeAfterCommitDigest: scan.lastScannedCommitDigest
});
- // Two ways a resume point stops meaning anything: a rewritten history (force push, or a
- // rebase landing between runs) takes the commit out of the repository entirely, or a newly
- // reachable ref puts commits ahead of it that this scan has never looked at. Either way the
- // repository is re-walked from the start rather than resumed past commits nobody scanned.
- if (scan.lastScannedCommit && !plan.resumed) {
- logger.warn(
- `secretScanningV2Queue: Full Scan cannot resume, restarting ${logDetails} [lastScannedCommit=${scan.lastScannedCommit}] [reason=${plan.prefixChanged ? "history before the resume point changed" : "resume point is no longer in the repository"}]`
- );
- }
-
logger.info(
`secretScanningV2Queue: Full Scan Planned ${logDetails} totalCommits=[${plan.totalCommits}] batches=[${plan.batches.length}] batchSize=[${batchSize}] resumed=[${plan.resumed}]`
);
@@ -512,21 +522,9 @@ export const secretScanningV2QueueServiceFactory = ({
if (retryCount === retryLimit) {
const errorMessage = parseScanErrorMessage(error);
- // Only a scan this run still owns is closed out here. A failure before the status was set
- // to `scanning` leaves the row `queued`, which is why that state is accepted too — but a
- // scan the reaper has already failed and notified on is left exactly as it is.
- const failedScans = await secretScanningV2DAL.scans.update(
- {
- id: scanId,
- $in: { status: [SecretScanningScanStatus.Queued, SecretScanningScanStatus.Scanning] }
- },
- {
- status: SecretScanningScanStatus.Failed,
- statusMessage: errorMessage
- }
- );
+ const failed = await markScanAsFailed({ scanId, statusMessage: errorMessage });
- if (failedScans.length) {
+ if (failed) {
await queueService.queue(
QueueName.SecretScanningV2,
QueueJobs.SecretScanningV2SendNotification,
@@ -667,20 +665,9 @@ export const secretScanningV2QueueServiceFactory = ({
);
try {
- // Same guard as the full scan: a retry after a post-completion side effect failed must not
- // reopen a scan that already reached a terminal status.
- const startedScans = await secretScanningV2DAL.scans.update(
- {
- id: scanId,
- $in: { status: [SecretScanningScanStatus.Queued, SecretScanningScanStatus.Scanning] }
- },
- {
- status: SecretScanningScanStatus.Scanning,
- scanningStartedAt: new Date()
- }
- );
+ const started = await markScanAsScanning({ scanId });
- if (!startedScans.length) {
+ if (!started) {
logger.warn(
`secretScanningV2Queue: Diff Scan skipped, scan was already closed out ${logDetails} [scanType=${SecretScanningScanType.DiffScan}]`
);
@@ -799,20 +786,9 @@ export const secretScanningV2QueueServiceFactory = ({
if (retryCount === retryLimit) {
const errorMessage = parseScanErrorMessage(error);
- // Same guard as the full scan: `queued` covers a failure before the scan started, and a
- // scan the reaper already closed out is left alone.
- const failedScans = await secretScanningV2DAL.scans.update(
- {
- id: scanId,
- $in: { status: [SecretScanningScanStatus.Queued, SecretScanningScanStatus.Scanning] }
- },
- {
- status: SecretScanningScanStatus.Failed,
- statusMessage: errorMessage
- }
- );
+ const failed = await markScanAsFailed({ scanId, statusMessage: errorMessage });
- if (failedScans.length) {
+ if (failed) {
await queueService.queue(
QueueName.SecretScanningV2,
QueueJobs.SecretScanningV2SendNotification,