Skip to content

OpenAI's Codex Security CLI and TypeScript SDK for finding, validating, and fixing security vulnerabilities. npm: https://www.npmjs.com/package/@openai/codex-security

fix(deep-scan): preserve distinct findings and bound duplicate comparisons (#2313)

Michael D'Angelo7ba1553804d7
diff --git a/plugins/codex-security/mcp-app/server.ts b/plugins/codex-security/mcp-app/server.ts
index 1cf3c592f..88f5aed47 100644
--- a/plugins/codex-security/mcp-app/server.ts
+++ b/plugins/codex-security/mcp-app/server.ts
@@ -36,7 +36,10 @@ import {
} from "./src/deep-scan/parent-sandbox.js";
import { WorkbenchDeepScanStore } from "./src/deep-scan/store.js";
import { readDeepScanProjectConfig } from "./src/deep-scan/project-config.js";
-import type { DeepScanRunState } from "./src/deep-scan/types.js";
+import type {
+ DeepScanLogEvent,
+ DeepScanRunState,
+} from "./src/deep-scan/types.js";
import { PLUGIN_ROOT, runWorkbench, scanRoot } from "./src/workbench-client.js";
const USER_INPUT_WAIT_TIMEOUT_MS = 14 * 60 * 1000;
@@ -2364,20 +2367,7 @@ function deepScanTerminalResult(
return undefined;
}
-function logDeepScanEvent(event: {
- event: string;
- scanId: string;
- workerId?: string;
- kind?: string;
- attempt?: number;
- count?: number;
- completed?: number;
- newFindings?: number;
- pass?: number;
- reason?: string;
- threadId?: string;
- total?: number;
-}): void {
+function logDeepScanEvent(event: DeepScanLogEvent): void {
console.error(
JSON.stringify({ component: "codex_security_deep_scan", ...event }),
);
diff --git a/plugins/codex-security/mcp-app/src/deep-scan/coordinator.ts b/plugins/codex-security/mcp-app/src/deep-scan/coordinator.ts
index c1a18ced4..cfa33feed 100644
--- a/plugins/codex-security/mcp-app/src/deep-scan/coordinator.ts
+++ b/plugins/codex-security/mcp-app/src/deep-scan/coordinator.ts
@@ -729,6 +729,9 @@ export class DeepScanCoordinator {
this.artifacts.dedupRoot,
"dedup",
);
+ let pass = persisted.filter(
+ (w) => w.kind === "dedup" && w.status === "succeeded",
+ ).length;
let dispatched = this.state.dispatchedCount;
let streak = this.state.noNewStreak;
let errors = this.state.consecutiveErrors;
@@ -756,7 +759,8 @@ export class DeepScanCoordinator {
signal.throwIfAborted();
if (batch.length) {
// Newly dispatched scans arrive in launch order. Recovery uses the same labels.
- for (const count of await this.mergeAcceptedBatch(result, batch)) {
+ const newCounts = await this.mergeAcceptedBatch(result, batch);
+ for (const count of newCounts) {
streak = count ? 0 : streak + 1;
}
resultPath = join(
@@ -768,8 +772,9 @@ export class DeepScanCoordinator {
"result.json",
);
await writeJsonAtomic(resultPath, persistedScanUnion(result));
+ const iterationId = randomUUID();
const saved = await this.options.store.recordIteration({
- id: randomUUID(),
+ id: iterationId,
scanId,
workerIds: batch.map((w) => w.id),
resultPath,
@@ -778,6 +783,15 @@ export class DeepScanCoordinator {
});
signal.throwIfAborted();
this.state = saved;
+ this.log({
+ event: "dedup_committed",
+ scanId,
+ workerId: iterationId,
+ pass: ++pass,
+ count: batch.length,
+ newFindings: newCounts.reduce((sum, count) => sum + count, 0),
+ retainedFindings: result.findings.length,
+ });
this.logProgress(acceptedCount);
}
if (errors >= config.stopAfterConsecutiveErrors && failure) {
diff --git a/plugins/codex-security/mcp-app/src/deep-scan/iteration.ts b/plugins/codex-security/mcp-app/src/deep-scan/iteration.ts
index 8d552c8d0..1f17e564e 100644
--- a/plugins/codex-security/mcp-app/src/deep-scan/iteration.ts
+++ b/plugins/codex-security/mcp-app/src/deep-scan/iteration.ts
@@ -2,6 +2,7 @@ import {
scanDraftInputSchema,
type ScanDraftInput,
} from "../artifact-scan-draft.js";
+import { z } from "zod";
import { asRecord } from "../record.js";
import { addScanCoverage, type ScanUnion } from "./artifact-validation.js";
import type { AcceptedDiscovery } from "./worker-runner.js";
@@ -201,6 +202,10 @@ function hasAdditionalControl(a: Finding, b: Finding) {
);
}
+function separateControls(a: Finding, b: Finding) {
+ return hasAdditionalControl(a, b) || hasAdditionalControl(b, a);
+}
+
function sourcesNeedSeparateReports(a: Finding, b: Finding) {
return a.provenance.sourceFindings?.some((left, i) => {
const worker = left.id.slice(0, left.id.lastIndexOf(":") + 1);
@@ -210,8 +215,7 @@ function sourcesNeedSeparateReports(a: Finding, b: Finding) {
if (right.id.startsWith(worker) && differentWorkerSites(source, other))
return true;
return (
- hasAdditionalControl(source, other) ||
- hasAdditionalControl(other, source) ||
+ separateControls(source, other) ||
((i > 0 || j > 0) && separateSinks(source, other))
);
});
@@ -245,7 +249,7 @@ function distinctIdentity(
);
}
-function addLocations(finding: Finding, source: Finding) {
+function addLocations(finding: Finding, source: Pick<Finding, "locations">) {
for (const location of source.locations) {
if (
!finding.locations.some(
@@ -262,31 +266,114 @@ function addLocations(finding: Finding, source: Finding) {
// Keep the original repair groups as comparison inputs even after two reports
// are published as one. A later batch must see the same reports after a resume.
-type RepairGroup = { order: number; sources: string[] };
+const repairGroupSchema = z.strictObject({
+ order: z.number().int().nonnegative(),
+ sources: z.array(z.string()),
+});
+const repairGroupsSchema = z.array(repairGroupSchema).min(1);
+type RepairGroup = z.infer<typeof repairGroupSchema>;
+
+function sourceIds(finding: Finding) {
+ return (
+ finding.provenance.sourceFindingIds ??
+ finding.provenance.sourceFindings?.map(({ id }) => id) ??
+ []
+ );
+}
+
+// The persisted schema accepts partial original records from older reducers.
+// Only groups with all their original reports can enter semantic comparisons.
+function hasCompleteSources(finding: Finding) {
+ const ids = sourceIds(finding);
+ const sources = finding.provenance.sourceFindings ?? [];
+ const known = new Set(sources.map(({ id }) => id));
+ return (
+ ids.length > 0 &&
+ ids.length === sources.length &&
+ known.size === ids.length &&
+ ids.every((id) => known.has(id)) &&
+ sources.every(
+ ({ finding }) =>
+ scanDraftInputSchema.shape.findings.element.safeParse(finding).success,
+ )
+ );
+}
+
function restoreRepairGroups(findings: Finding[]) {
- const groups: { finding: Finding; target: number }[] = [];
- for (const [id, visible] of findings.entries()) {
- const parts = visible.provenance.deepScanRepairGroups as
- RepairGroup[] | undefined;
- if (!parts) {
- groups[id] = { finding: visible, target: id };
+ function visibleGroups() {
+ // Pre-rewrite reducers could write arbitrary fields in provenance.
+ return findings.map((finding, target) => {
+ delete finding.provenance.deepScanRepairGroups;
+ return { finding, target };
+ });
+ }
+ if (findings[0]?.provenance.deepScanRepairGroups === undefined)
+ return visibleGroups();
+
+ const groups = new Map<number, { finding: Finding; target: number }>();
+ let previousTarget = -1;
+ for (const visible of findings) {
+ const parts = repairGroupsSchema.safeParse(
+ visible.provenance.deepScanRepairGroups,
+ ).data;
+ if (!parts || parts[0]!.order <= previousTarget) return visibleGroups();
+ const target = parts[0]!.order;
+ previousTarget = target;
+ if (parts.length === 1) {
+ // Singletons already carry the visible report, even if a legacy source
+ // has only a title or a reference with no archived finding.
+ const ids = sourceIds(visible);
+ const sources = parts[0]!.sources;
+ if (
+ groups.has(target) ||
+ ids.length !== sources.length ||
+ !ids.every((id, i) => id === sources[i])
+ )
+ return visibleGroups();
+ delete visible.provenance.deepScanRepairGroups;
+ groups.set(target, { finding: visible, target });
continue;
}
const originals = new Map(
- visible.provenance.sourceFindings!.map((source) => [source.id, source]),
+ visible.provenance.sourceFindings?.map((source) => [source.id, source]),
);
for (const { order, sources } of parts) {
- const members = sources.map((id) => originals.get(id)!);
- const finding = structuredClone(members[0]!.finding) as Finding;
+ if (!sources.length || order < target || groups.has(order))
+ return visibleGroups();
+ const members = [];
+ for (const id of sources) {
+ const source = originals.get(id);
+ if (!source) return visibleGroups();
+ originals.delete(id);
+ members.push(source);
+ }
+ const finding = scanDraftInputSchema.shape.findings.element.safeParse(
+ members[0]!.finding,
+ ).data;
+ if (!finding) return visibleGroups();
delete finding.provenance.deepScanRepairGroups;
finding.provenance.sourceFindingIds = sources;
finding.provenance.sourceFindings = members;
- for (const member of members.slice(1))
- addLocations(finding, member.finding as Finding);
- groups[order] = { finding, target: parts[0]!.order };
+ for (const member of members.slice(1)) {
+ const locations =
+ scanDraftInputSchema.shape.findings.element.shape.locations.safeParse(
+ member.finding.locations,
+ ).data;
+ if (!locations) return visibleGroups();
+ addLocations(finding, { locations });
+ }
+ groups.set(order, { finding, target });
}
+ if (originals.size) return visibleGroups();
+ }
+ // Validate the dense sequence before assigning any array indexes.
+ const ordered = [];
+ for (let order = 0; order < groups.size; order++) {
+ const group = groups.get(order);
+ if (!group) return visibleGroups();
+ ordered.push(group);
}
- return groups;
+ return ordered;
}
export async function mergeBatch(
@@ -311,8 +398,15 @@ export async function mergeBatch(
JSON.stringify({ ...finding, provenance: undefined });
const exact = originals.map(content);
const copies = new Map<string, (typeof retained)[number]>();
+ const incomplete = new Set<number>();
for (const group of retained) {
copies.set(exact[group.id]!, group);
+ if (!hasCompleteSources(group.finding)) {
+ incomplete.add(group.target);
+ continue;
+ }
+ // A legacy checkpoint can include all originals without the separate ID array.
+ group.finding.provenance.sourceFindingIds ??= sourceIds(group.finding);
for (const source of group.finding.provenance.sourceFindings ?? [])
copies.set(content(source.finding), group);
}
@@ -352,6 +446,7 @@ export async function mergeBatch(
retained.find((prior) => {
const other = repairs?.get(prior.id);
if (
+ incomplete.has(prior.target) ||
!key ||
!other ||
key.length !== other.length ||
@@ -375,7 +470,7 @@ export async function mergeBatch(
source,
finding,
id.startsWith(worker.id + ":"),
- ),
+ ) || separateControls(source, finding as Finding),
)
);
});
@@ -394,7 +489,9 @@ export async function mergeBatch(
const sourceId = `${worker.id}:${index}`;
copies.set(exact[id]!, group);
const { finding } = group;
- (finding.provenance.sourceFindingIds ??= []).push(sourceId);
+ (finding.provenance.sourceFindingIds ??= sourceIds(finding)).push(
+ sourceId,
+ );
(finding.provenance.sourceFindings ??= []).push({
id: sourceId,
finding: structuredClone(source),
@@ -423,43 +520,40 @@ export async function mergeBatch(
group.target = group.id;
group.worker = worker;
}
- if (
- repairs &&
- retained.some((group) => group.worker >= 0) &&
- retained.every(
- ({ finding }) =>
- finding.provenance.sourceFindings?.length &&
- finding.provenance.sourceFindingIds?.length,
- )
- ) {
- let matches: Map<number, Set<number>> | undefined;
- try {
- const sources = new Map<number, Record<string, unknown>[]>();
- for (const group of retained) {
- const members = sources.get(group.target) ?? [];
- members.push(
- ...group.finding.provenance.sourceFindings!.map((s) => s.finding),
- );
- sources.set(group.target, members);
- }
- matches = await checkSameFix(
- retained.filter((group) => group.target === group.id),
- priorTargets,
- records,
- repairs,
- sources,
- compare,
+ if (repairs && retained.some((group) => group.worker >= 0)) {
+ const sources = new Map<number, Record<string, unknown>[]>();
+ for (const group of retained) {
+ if (incomplete.has(group.target)) continue;
+ const members = sources.get(group.target) ?? [];
+ members.push(
+ ...group.finding.provenance.sourceFindings!.map((s) => s.finding),
);
- } catch (error) {
- onError(error);
+ sources.set(group.target, members);
+ }
+ const reviewFailures = new Set<string>();
+ const matches = await checkSameFix(
+ retained.filter(
+ (group) => group.target === group.id && !incomplete.has(group.target),
+ ),
+ priorTargets,
+ records,
+ repairs,
+ sources,
+ compare,
+ (error) => {
+ onError(error);
+ reviewFailures.add(errorMessage(error));
+ },
+ );
+ if (reviewFailures.size) {
limitation = [
limitation,
- `Deep Scan could not check disputed duplicates: ${errorMessage(error)}`,
+ `Deep Scan could not check disputed duplicates: ${[...reviewFailures].join(" / ")}`,
]
.filter(Boolean)
.join(" ");
}
- if (matches?.size) {
+ if (matches.size) {
const kept = new Map<number, typeof retained>();
for (const group of retained) {
const earlier = matches.get(group.id);
@@ -505,7 +599,7 @@ export async function mergeBatch(
}
(target.provenance.deepScanRepairGroups as RepairGroup[]).push({
order,
- sources: group.finding.provenance.sourceFindingIds!,
+ sources: sourceIds(group.finding),
});
}
aggregate.findings = [...visible.values()];
diff --git a/plugins/codex-security/mcp-app/src/deep-scan/same-fix.ts b/plugins/codex-security/mcp-app/src/deep-scan/same-fix.ts
index 1963d227e..002da54cb 100644
--- a/plugins/codex-security/mcp-app/src/deep-scan/same-fix.ts
+++ b/plugins/codex-security/mcp-app/src/deep-scan/same-fix.ts
@@ -59,6 +59,8 @@ const sourceInstructions = `Judge each pair of original reports independently as
Identify the faulty operation or missing check and the smallest change that fixes A. Do the same for B. Read the summary, root cause and code. A missing check that a report explicitly describes as a cause counts even if it has the same impact as the primary bug. Suggested hardening and different ways of implementing the correction do not count as additional bugs.
Apply each fix alone. If a claimed defect in the other report survives, quote the phrase that claims it and name a concrete input or operation that still fails. Put these counterexamples in survivingPaths. Different examples, callers, or consequences of one faulty operation are compatible when correcting that operation resolves both. Do not infer an additional requirement from a hypothetical attack that neither report claims. Return same only when the two complete corrections are equivalent and the evidence supports the decision. Keep uncertain pairs separate.`;
+// Each pair needs its own model verdict and two evidence references.
+const PAIRS_PER_REVIEW = 16;
type Pair = { id: number; left: number; right: number };
function claim({
title,
@@ -71,27 +73,22 @@ function claim({
return { title, summary, rootCause, remediation, locations, codeEvidence };
}
+// Structured output constrains each row, but can still return the same ID twice.
+function unambiguous<T>(rows: T[], id: (row: T) => number): T[] {
+ return [...Map.groupBy(rows, id).values()].flatMap((group) =>
+ group.length === 1 ? group : [],
+ );
+}
+
async function confirmSources(
pairs: Pair[],
- originals: Map<number, Record<string, unknown>[]>,
+ claimsFor: (id: number) => ReturnType<typeof claim>[],
compare: CompareReports,
) {
- const cases = pairs.flatMap((pair) =>
- originals
- .get(pair.right)!
- .flatMap((a) =>
- originals
- .get(pair.left)!
- .map((b) => ({ pair, A: claim(a), B: claim(b) })),
- ),
- );
const rejected = new Set<number>();
- // Grade small independent pairs. A grouped report can carry many original claims.
- for (let start = 0; start < cases.length; start += 4) {
- const batch = cases
- .slice(start, start + 4)
- .filter(({ pair }) => !rejected.has(pair.id));
- if (!batch.length) continue;
+ type Claim = ReturnType<typeof claim>;
+ const batch: { pair: Pair; A: Claim; B: Claim }[] = [];
+ async function gradeBatch() {
const rows = batch.map(({ A, B }, id) => ({ id, A, B }));
const { sources } = JSON.parse(
await compare(
@@ -100,16 +97,31 @@ async function confirmSources(
),
);
const same = new Set(
- sources
- .filter(
- (s: { same: boolean; survivingPaths: string[] }) =>
- s.same && !s.survivingPaths.length,
- )
- .map((s: { id: number }) => s.id),
+ unambiguous(
+ sources as { id: number; same: boolean; survivingPaths: string[] }[],
+ (s) => s.id,
+ )
+ .filter((s) => s.same && !s.survivingPaths.length)
+ .map((s) => s.id),
);
for (const [id, { pair }] of batch.entries())
if (!same.has(id)) rejected.add(pair.id);
+ batch.length = 0;
}
+ // A single surviving path rejects the pair. Generate only the cases we still need.
+ nextPair: for (const pair of pairs) {
+ const left = claimsFor(pair.left);
+ for (const A of claimsFor(pair.right)) {
+ for (const B of left) {
+ batch.push({ pair, A, B });
+ if (batch.length === 4) {
+ await gradeBatch();
+ if (rejected.has(pair.id)) continue nextPair;
+ }
+ }
+ }
+ }
+ if (batch.length) await gradeBatch();
return pairs.filter(({ id }) => !rejected.has(id));
}
@@ -138,80 +150,128 @@ export async function checkSameFix(
repairs: Map<number, number[] | undefined>,
originals: Map<number, Record<string, unknown>[]>,
compare: CompareReports,
+ onReviewError: (error: unknown) => void,
) {
const evidence = records.map((record) =>
record.evidence.flatMap(([path, code]) => (path ? [{ path, code }] : [])),
);
- const batches = new Map<number, Pair[]>();
- for (const [i, { id: right, worker }] of groups.entries()) {
- if (worker < 0) continue;
- const b = repairs.get(right);
- for (const { id: left } of groups.slice(0, i)) {
- const a = repairs.get(left);
- const prior = priorTargets[right];
- if (prior !== undefined && priorTargets[left] !== prior) continue;
- if (
- prior === undefined &&
- !(a && b
- ? a.some((key) => b.includes(key))
- : sharesSource(records[left]!, records[right]!))
- )
- continue;
- if (
- !evidence[left]!.some((x) =>
- evidence[right]!.some((y) => x.path === y.path),
+ const pending = groups
+ .map(({ id, worker }, index) => ({ id, worker, index }))
+ .filter(({ worker }) => worker >= 0);
+ const workers = Map.groupBy(pending, ({ worker }) => worker);
+ function* candidatePairs(workerGroups: typeof pending): Generator<Pair> {
+ let pairId = 0;
+ for (const { id: right, index } of workerGroups) {
+ const b = repairs.get(right);
+ for (let i = 0; i < index; i++) {
+ const left = groups[i]!.id;
+ const a = repairs.get(left);
+ const prior = priorTargets[right];
+ if (prior !== undefined && priorTargets[left] !== prior) continue;
+ if (
+ prior === undefined &&
+ !(a && b
+ ? a.some((key) => b.includes(key))
+ : sharesSource(records[left]!, records[right]!))
)
- )
- continue;
- const pairs = batches.get(worker) ?? [];
- pairs.push({ id: pairs.length, left, right });
- batches.set(worker, pairs);
+ continue;
+ if (
+ !evidence[left]!.some((x) =>
+ evidence[right]!.some((y) => x.path === y.path),
+ )
+ )
+ continue;
+ yield { id: pairId++, left, right };
+ }
}
}
+ const claims = new Map<number, ReturnType<typeof claim>[]>();
+ function claimsFor(id: number) {
+ let rows = claims.get(id);
+ if (rows === undefined) {
+ // Keep every observation in the artifact. Send identical evidence to the model once.
+ const distinct = new Map<string, ReturnType<typeof claim>>();
+ for (const source of originals.get(id)!) {
+ const row = claim(source);
+ distinct.set(JSON.stringify(row), row);
+ }
+ rows = [...distinct.values()];
+ claims.set(id, rows);
+ }
+ return rows;
+ }
const matches = new Map<number, Set<number>>();
+ async function reviewPairs(pairs: Pair[]) {
+ const reports = [
+ ...new Set(pairs.flatMap(({ left, right }) => [left, right])),
+ ].map((id) => {
+ const [first, ...rest] = claimsFor(id);
+ const { title, summary, rootCause, remediation } = first!;
+ return {
+ id,
+ primary: {
+ title,
+ summary,
+ rootCause,
+ remediation,
+ locations: records[id]!.locations,
+ evidence: evidence[id]!.map((excerpt, id) => ({ id, ...excerpt })),
+ },
+ attached: rest,
+ };
+ });
+ const { reviews } = JSON.parse(
+ await compare(
+ instructions + "\n" + JSON.stringify({ pairs, reports }),
+ schema,
+ ),
+ );
+ const proposed: Pair[] = [];
+ const decisions = unambiguous(
+ reviews as {
+ pair: number;
+ decision: string;
+ leftEvidence: number;
+ rightEvidence: number;
+ }[],
+ (review) => review.pair,
+ );
+ for (const { pair, decision, leftEvidence, rightEvidence } of decisions) {
+ const candidate = pairs.find(({ id }) => id === pair);
+ if (decision !== "same" || !candidate) continue;
+ const { left, right } = candidate;
+ const path = evidence[left]![leftEvidence]?.path;
+ if (!path || path !== evidence[right]![rightEvidence]?.path) continue;
+ proposed.push(candidate);
+ }
+ for (const { left, right } of await confirmSources(
+ proposed,
+ claimsFor,
+ compare,
+ )) {
+ const earlier = matches.get(right) ?? new Set<number>();
+ earlier.add(left);
+ matches.set(right, earlier);
+ }
+ }
+ async function reviewBatch(pairs: Pair[]) {
+ try {
+ await reviewPairs(pairs);
+ } catch (error) {
+ onReviewError(error);
+ }
+ }
const results = await Promise.allSettled(
- [...batches.values()].map(async (pairs) => {
- const reports = [
- ...new Set(pairs.flatMap(({ left, right }) => [left, right])),
- ].map((id) => {
- const [first, ...rest] = originals.get(id)!;
- const { title, summary, rootCause, remediation } = first!;
- return {
- id,
- primary: {
- title,
- summary,
- rootCause,
- remediation,
- locations: records[id]!.locations,
- evidence: evidence[id]!.map((excerpt, id) => ({ id, ...excerpt })),
- },
- attached: rest.map(claim),
- };
- });
- const { reviews } = JSON.parse(
- await compare(
- instructions + "\n" + JSON.stringify({ pairs, reports }),
- schema,
- ),
- );
- const proposed: Pair[] = [];
- for (const { pair, decision, leftEvidence, rightEvidence } of reviews) {
- if (decision !== "same" || !pairs[pair]) continue;
- const { left, right } = pairs[pair]!;
- const path = evidence[left]![leftEvidence]?.path;
- if (!path || path !== evidence[right]![rightEvidence]?.path) continue;
- proposed.push(pairs[pair]!);
- }
- for (const { left, right } of await confirmSources(
- proposed,
- originals,
- compare,
- )) {
- const earlier = matches.get(right) ?? new Set<number>();
- earlier.add(left);
- matches.set(right, earlier);
+ [...workers.values()].map(async (workerGroups) => {
+ const batch: Pair[] = [];
+ for (const pair of candidatePairs(workerGroups)) {
+ batch.push(pair);
+ if (batch.length === PAIRS_PER_REVIEW) {
+ await reviewBatch(batch);
+ batch.length = 0;
+ }
}
+ if (batch.length) await reviewBatch(batch);
}),
);
for (const result of results)
diff --git a/plugins/codex-security/mcp-app/src/deep-scan/types.ts b/plugins/codex-security/mcp-app/src/deep-scan/types.ts
index c1ac71b71..fb7b8c223 100644
--- a/plugins/codex-security/mcp-app/src/deep-scan/types.ts
+++ b/plugins/codex-security/mcp-app/src/deep-scan/types.ts
@@ -115,6 +115,7 @@ export interface DeepScanLogEvent {
count?: number;
completed?: number;
newFindings?: number;
+ retainedFindings?: number;
pass?: number;
reason?: string;
total?: number;
diff --git a/plugins/codex-security/mcp-app/tests/test_deep_scan_iterations.ts b/plugins/codex-security/mcp-app/tests/test_deep_scan_iterations.ts
index 281d8c992..c9fb0fb6c 100644
--- a/plugins/codex-security/mcp-app/tests/test_deep_scan_iterations.ts
+++ b/plugins/codex-security/mcp-app/tests/test_deep_scan_iterations.ts
@@ -719,16 +719,19 @@ test("fixing login cannot hide a separately cited permissive action policy", asy
{ path: "policy.py", startLine: 2, role: "control" },
],
};
- const { draft } = await scanLoop([[login, policy]], {
- maxDiscoveryRuns: 1,
- stopAfterNoNew: 1,
- comparisons: [[[0, 1], [1]]],
- sameFix: approvePairs,
- });
- assert.deepEqual(
- draft.findings.map((f) => f.title),
- [login.title, policy.title],
- );
+ // The primary comparison can agree on a single repair or send the pair to confirmation.
+ for (const assignments of [[[0, 1]], [[0, 1], [1]]]) {
+ const { draft } = await scanLoop([[login, policy]], {
+ maxDiscoveryRuns: 1,
+ stopAfterNoNew: 1,
+ comparisons: [assignments],
+ sameFix: approvePairs,
+ });
+ assert.deepEqual(
+ draft.findings.map((f) => f.title),
+ [login.title, policy.title],
+ );
+ }
});
test("a later source with a surviving path reopens the merge and counts as new", async () => {
@@ -780,6 +783,9 @@ test("a failed comparison waits for every started worker batch", async () => {
await release.promise;
return JSON.stringify({ reviews: [] });
},
+ (error: unknown) => {
+ throw error;
+ },
);
const rejected = assert.rejects(check, /comparison failed/).then(() => {
finished = true;
@@ -1010,3 +1016,351 @@ test("a merged report cannot lend its extra citation to a later comparison", asy
[2, 1],
);
});
+
+for (const conflict of ["reviews", "sources"]) {
+ test(`conflicting ${conflict} cannot hide an independent finding`, async () => {
+ const { checkSameFix } = await importSource(
+ join(import.meta.dirname, "../src/deep-scan/same-fix.ts"),
+ );
+ const groups = [
+ { id: 0, worker: 0 },
+ { id: 1, worker: 0 },
+ ];
+ const records = groups.map(() => ({
+ locations: [["fixture.py", 1, 1]],
+ evidence: [["fixture.py", "lookup(id)"]],
+ }));
+ const matches = await checkSameFix(
+ groups,
+ [],
+ records,
+ new Map(),
+ new Map(groups.map(({ id }) => [id, [lookup]])),
+ async (
+ _prompt: string,
+ schema: { properties: Record<string, unknown> },
+ ) => {
+ if ("reviews" in schema.properties) {
+ const approve = samePair(0);
+ return JSON.stringify({
+ reviews:
+ conflict === "reviews"
+ ? [approve, { ...approve, decision: "keep" }]
+ : [approve],
+ });
+ }
+ const approve = {
+ id: 0,
+ same: true,
+ fixA: "validate",
+ fixB: "validate",
+ survivingPaths: [],
+ };
+ return JSON.stringify({
+ sources:
+ conflict === "sources"
+ ? [
+ approve,
+ {
+ ...approve,
+ same: false,
+ survivingPaths: ["second handler"],
+ },
+ ]
+ : [approve],
+ });
+ },
+ (error: unknown) => {
+ throw error;
+ },
+ );
+ assert.equal(matches.size, 0);
+ });
+}
+
+test("repeated original observations require one source comparison", async () => {
+ const observations = Array.from({ length: 8 }, () => lookup);
+ let sourceChecks = 0;
+ const alias = {
+ ...lookup,
+ title: "Another description of the shared lookup",
+ };
+ const { draft } = await scanLoop([[...observations, alias]], {
+ maxDiscoveryRuns: 1,
+ stopAfterNoNew: 1,
+ comparisons: [
+ () => ({
+ uncertain: observations.map((_, i) => i).concat(observations.length),
+ defects: [],
+ }),
+ ],
+ sameFix: approvePairs,
+ sameSource: () => {
+ sourceChecks++;
+ return true;
+ },
+ });
+ assert.equal(draft.findings.length, 1);
+ assert.equal(draft.findings[0]!.provenance.sourceFindings!.length, 9);
+ assert.equal(sourceChecks, 1);
+});
+
+test("review batches limit pair counts and isolate rejected and stale decisions", async () => {
+ const { checkSameFix } = await importSource(
+ join(import.meta.dirname, "../src/deep-scan/same-fix.ts"),
+ );
+ const groups = Array.from({ length: 9 }, (_, id) => ({ id, worker: 0 }));
+ const records = groups.map(() => ({
+ locations: [["fixture.py", 1, 1]],
+ evidence: [["fixture.py", "lookup(id)"]],
+ }));
+ const sizes: number[] = [];
+ const failures: unknown[] = [];
+ const expected = new Map<number, Set<number>>();
+ const matches = await checkSameFix(
+ groups,
+ [],
+ records,
+ new Map(),
+ new Map(groups.map(({ id }) => [id, [lookup]])),
+ async (prompt: string, schema: { properties: Record<string, unknown> }) => {
+ const input = JSON.parse(prompt.split("\n").at(-1)!);
+ if ("reviews" in schema.properties) {
+ const { pairs } = input as PairCheck;
+ sizes.push(pairs.length);
+ if (sizes.length === 1) throw new Error("review failed");
+ const last = pairs.at(-1)!;
+ const earlier = expected.get(last.right) ?? new Set<number>();
+ earlier.add(last.left);
+ expected.set(last.right, earlier);
+ // ID zero belongs to the failed first batch.
+ return JSON.stringify({ reviews: [samePair(0), samePair(last.id)] });
+ }
+ return JSON.stringify({
+ sources: input.map(({ id }: { id: number }) => ({
+ id,
+ same: true,
+ survivingPaths: [],
+ })),
+ });
+ },
+ (error: unknown) => failures.push(error),
+ );
+ assert.deepEqual(sizes, [16, 16, 4]);
+ assert.deepEqual(matches, expected);
+ assert.equal(failures.length, 1);
+});
+
+test("legacy provenance cannot drop or overwrite visible findings on resume", async () => {
+ const { mergeBatch } = await importSource(
+ join(import.meta.dirname, "../src/deep-scan/iteration.ts"),
+ );
+ const { parseScanUnion } = await importSource(
+ join(import.meta.dirname, "../src/deep-scan/artifact-validation.ts"),
+ );
+ const findings = [lookup, { ...lookup, title: "Independent report" }];
+ for (const metadata of [
+ "old model annotation",
+ [],
+ [{ order: 0, sources: ["missing:0"] }],
+ [{ order: 2147483647, sources: ["w:0"] }],
+ [{ order: 0, sources: ["w:0"] }],
+ ]) {
+ const restored = parseScanUnion({
+ scanId: randomUUID(),
+ findings: findings.map((finding, i) => ({
+ ...finding,
+ provenance: {
+ ...finding.provenance,
+ sourceFindingIds: [`w:${i}`],
+ sourceFindings: [{ id: `w:${i}`, finding }],
+ // Older reducers could put arbitrary model-supplied fields here.
+ deepScanRepairGroups: i ? [{ order: 0, sources: ["w:1"] }] : metadata,
+ },
+ })),
+ });
+ const unexpected = () => {
+ throw new Error("unexpected model call");
+ };
+ await mergeBatch(restored, [], "/tmp", unexpected, unexpected);
+ assert.deepEqual(
+ restored.findings.map((f: typeof lookup) => f.title),
+ findings.map((f) => f.title),
+ );
+ assert.ok(
+ restored.findings.every(
+ (f: typeof lookup) => f.provenance.deepScanRepairGroups === undefined,
+ ),
+ );
+ }
+});
+
+test("resuming partial source ancestry isolates only the ambiguous finding", async () => {
+ const { mergeBatch } = await importSource(
+ join(import.meta.dirname, "../src/deep-scan/iteration.ts"),
+ );
+ const { parseScanUnion, persistedScanUnion, scanUnionToDraft } =
+ await importSource(
+ join(import.meta.dirname, "../src/deep-scan/artifact-validation.ts"),
+ );
+ for (const ancestry of [
+ {
+ sourceFindings: [{ id: "w:0", finding: { title: "old" } }],
+ sourceFindingIds: ["w:0"],
+ },
+ {
+ sourceFindings: [
+ { id: "w:0", finding: { title: "old", locations: lookup.locations } },
+ ],
+ },
+ { sourceFindingIds: ["w:0"] },
+ {},
+ ]) {
+ const scanId = randomUUID();
+ const old = {
+ ...lookup,
+ title: "Previously visible report",
+ provenance: { ...lookup.provenance, ...ancestry },
+ };
+ let saved = parseScanUnion({ scanId, findings: [old] });
+ const titles = ["Current bug", "Alias of current bug"];
+ const worker = (id: string, entries: string[]) => ({
+ id,
+ resultPath: join("/tmp", id, "result.json"),
+ result: {
+ ...standardScanDraft(scanId, undefined, id),
+ findings: entries.map((title) => ({ ...lookup, title })),
+ },
+ });
+ const fail = (error: unknown) => {
+ throw error;
+ };
+ let reviews = 0;
+ await mergeBatch(
+ saved,
+ [worker("second", titles)],
+ "/tmp",
+ async (
+ prompt: string,
+ schema: { properties: Record<string, unknown> },
+ ) => {
+ const input = JSON.parse(prompt.split("\n").at(-1)!);
+ if ("reviews" in schema.properties) {
+ reviews++;
+ assert.deepEqual(
+ input.pairs.map((p: { left: number; right: number }) => [
+ p.left,
+ p.right,
+ ]),
+ [[1, 2]],
+ );
+ return JSON.stringify(approvePairs(input));
+ }
+ if ("sources" in schema.properties)
+ return JSON.stringify({
+ sources: input.map(({ id }: { id: number }) => ({
+ id,
+ same: true,
+ survivingPaths: [],
+ })),
+ });
+ // The old report and the new primary share a proposed direct assignment;
+ // the alias also lists a second assignment and needs same-fix confirmation.
+ return JSON.stringify({
+ uncertain: [],
+ defects: [[0, 1, 2], [2]].map((ids) => ({
+ operation: "lookup",
+ failure: "check",
+ path: "fixture.py",
+ reports: ids.map((id) => ({ id, evidence: 0 })),
+ })),
+ });
+ },
+ fail,
+ );
+ assert.equal(reviews, 1);
+ saved = parseScanUnion(
+ JSON.parse(JSON.stringify(persistedScanUnion(saved))),
+ );
+ await mergeBatch(
+ saved,
+ [worker("third", [titles[1]!])],
+ "/tmp",
+ fail,
+ fail,
+ );
+ const result = scanUnionToDraft(saved);
+ assert.deepEqual(
+ result.findings.map((f: typeof lookup) => f.title),
+ [old.title, titles[0]],
+ );
+ assert.deepEqual(result.findings[0].provenance, old.provenance);
+ assert.deepEqual(result.findings[1].provenance.sourceFindingIds, [
+ "second:0",
+ "second:1",
+ "third:0",
+ ]);
+ }
+});
+
+for (const partialSource of ["primary", "secondary"] as const) {
+ test(`an exact repeat stays merged after reloading a partial ${partialSource} source`, async () => {
+ const { mergeBatch } = await importSource(
+ join(import.meta.dirname, "../src/deep-scan/iteration.ts"),
+ );
+ const { parseScanUnion, persistedScanUnion, scanUnionToDraft } =
+ await importSource(
+ join(import.meta.dirname, "../src/deep-scan/artifact-validation.ts"),
+ );
+ const scanId = randomUUID();
+ const sources = [lookup, lookup].map((finding, index) => ({
+ id: `old:${index}`,
+ finding:
+ index === (partialSource === "primary" ? 0 : 1)
+ ? { title: finding.title }
+ : finding,
+ }));
+ const checkpoint = {
+ scanId,
+ findings: [
+ {
+ ...lookup,
+ provenance: { ...lookup.provenance, sourceFindings: sources },
+ },
+ ],
+ };
+ const result = parseScanUnion(
+ JSON.parse(JSON.stringify(persistedScanUnion(checkpoint))),
+ );
+ const worker = {
+ id: "new",
+ resultPath: "/tmp/new/result.json",
+ result: {
+ ...standardScanDraft(scanId, undefined, "new"),
+ findings: parseScanUnion({ scanId, findings: [lookup] }).findings,
+ },
+ };
+ let comparisons = 0;
+ const compare = async () => {
+ comparisons++;
+ return JSON.stringify({ uncertain: [0, 1], defects: [] });
+ };
+ assert.deepEqual(
+ await mergeBatch(result, [worker], "/tmp", compare, () => {}),
+ [0],
+ );
+ const published = scanUnionToDraft(result).findings;
+ assert.equal(published.length, 1);
+ assert.equal(comparisons, 0);
+ assert.equal(published[0].title, lookup.title);
+ assert.deepEqual(published[0].provenance.sourceFindingIds, [
+ "old:0",
+ "old:1",
+ "new:0",
+ ]);
+ assert.deepEqual(
+ published[0].provenance.sourceFindings.slice(0, 2),
+ sources,
+ );
+ });
+}
diff --git a/plugins/codex-security/mcp-app/tests/test_deep_scan_logging.ts b/plugins/codex-security/mcp-app/tests/test_deep_scan_logging.ts
index a61c44608..d9603d7db 100644
--- a/plugins/codex-security/mcp-app/tests/test_deep_scan_logging.ts
+++ b/plugins/codex-security/mcp-app/tests/test_deep_scan_logging.ts
@@ -34,6 +34,20 @@ try {
assert.equal(terminal?.status, "succeeded", terminal?.error);
assert.equal(executor.discoveryCalls, 1);
assert.equal(store.iterations.length, 1);
+ assert.deepEqual(
+ events.filter(({ event }) => event === "dedup_committed"),
+ [
+ {
+ event: "dedup_committed",
+ scanId: fixture.run.scanId,
+ workerId: store.iterations[0]!.id,
+ pass: 1,
+ count: 1,
+ newFindings: 1,
+ retainedFindings: 1,
+ },
+ ],
+ );
for (const event of [
"coordinator_started",
"discovery_accepted",