Skip to content

OpenAI's Codex Security CLI and TypeScript SDK for finding, validating, and fixing security vulnerabilities. npm: https://www.npmjs.com/package/@openai/codex-security

fix(deep-scan): preserve worker coverage during recovery (#1064)

Michael D'Angelo8fd19efadb16
diff --git a/plugins/codex-security/mcp-app/src/artifact-deep-reducer.ts b/plugins/codex-security/mcp-app/src/artifact-deep-reducer.ts
index 90515b491..74f9fa607 100644
--- a/plugins/codex-security/mcp-app/src/artifact-deep-reducer.ts
+++ b/plugins/codex-security/mcp-app/src/artifact-deep-reducer.ts
@@ -1,7 +1,8 @@
-import { join } from "node:path";
+import { dirname, join, posix, relative, sep } from "node:path";
import type { ArtifactContext } from "./artifact-context.js";
import {
parsePersistedScanDraft,
+ retainRecordedReceiptOwnership,
saveScanDraftCheckpoint,
} from "./artifact-scan-draft.js";
import { artifactSchemaDocuments } from "./artifact-contracts.js";
@@ -13,7 +14,9 @@ import {
writeJsonAtomic,
} from "./deep-scan/artifacts.js";
import {
+ deepReductionForPersistence,
parseDeepReduction,
+ projectDiscoveryCoverage,
parseStoredScanDraft,
reconcileDeepReduction,
type DeepReductionInput,
@@ -27,6 +30,20 @@ export { deepReductionInputSchema } from "./deep-scan/artifact-validation.js";
/** Read the findings and scan context assigned to this reducer. */
export async function getCodexSecurityDeepReducerInputs(
context: ArtifactContext,
+): Promise<DeepReductionSources> {
+ const inputs = await readDeepReductionSources(context);
+ return {
+ discoveries: inputs.discoveries.map(({ workerId, result }) => ({
+ workerId,
+ result,
+ })),
+ previous: inputs.previous,
+ };
+}
+
+/** Snapshot accepted source review separately from the reducer's model inputs. */
+export async function readDeepReductionSources(
+ context: ArtifactContext,
): Promise<DeepReductionSources> {
return withLogicalReducerErrors(context, async () => {
const bound = bindDeepReducer(context);
@@ -36,8 +53,13 @@ export async function getCodexSecurityDeepReducerInputs(
worker.resultPath,
bound.artifacts.workersRoot,
);
+ const stored = await readJsonObject(worker.resultPath);
+ await retainRecordedReceiptOwnership(
+ { ...context, root: dirname(worker.resultPath), layout: "worker" },
+ stored,
+ );
const result = parseStoredScanDraft(
- await readJsonObject(worker.resultPath),
+ stored,
"Accepted Standard worker " + worker.id,
bound.scanId,
parsePersistedScanDraft,
@@ -52,8 +74,60 @@ export async function getCodexSecurityDeepReducerInputs(
const provenance = finding.provenance as Record<string, unknown>;
provenance.sourceFindingIds = [`${worker.id}:${index}`];
}
- const { coverage: _coverage, ...reduction } = result;
- return { workerId: worker.id, result: reduction };
+ const { coverage, ...reduction } = result;
+ const scanReceiptRefs = new Map<number, ReadonlySet<string>>();
+ for (const [index, surface] of (
+ coverage.surfaces as Record<string, unknown>[]
+ ).entries()) {
+ const surfaceScanRefs = new Set<string>();
+ scanReceiptRefs.set(index, surfaceScanRefs);
+ for (const ref of (surface.receiptRefs as string[] | undefined) ??
+ []) {
+ const normalized = posix.normalize(ref);
+ const inheritedScanRefs = (
+ surface.provenance as Record<string, unknown> | undefined
+ )?.scanReceiptRefs;
+ if (
+ Array.isArray(inheritedScanRefs) &&
+ inheritedScanRefs.includes(normalized)
+ ) {
+ surfaceScanRefs.add(normalized);
+ continue;
+ }
+ try {
+ await requireRegularFile(
+ join(dirname(worker.resultPath), normalized),
+ dirname(worker.resultPath),
+ true,
+ );
+ continue;
+ } catch {
+ // Shared receipts are a fallback when no valid worker-local file was found.
+ }
+ try {
+ await requireRegularFile(
+ join(bound.artifacts.scanDir, normalized),
+ bound.artifacts.scanDir,
+ true,
+ );
+ surfaceScanRefs.add(normalized);
+ } catch {
+ // Worker-local receipts are qualified below; finalization validates evidence.
+ }
+ }
+ }
+ return {
+ workerId: worker.id,
+ coverage: projectDiscoveryCoverage(
+ coverage,
+ worker,
+ relative(bound.artifacts.scanDir, dirname(worker.resultPath))
+ .split(sep)
+ .join("/"),
+ scanReceiptRefs,
+ ),
+ result: reduction,
+ };
}),
);
const previous = await readPreviousReduction(bound);
@@ -88,7 +162,7 @@ export async function recordCodexSecurityDeepReduction(
throw new Error(
"Deep reduction is only a checkpoint, not a complete result.",
);
- const inputs = await getCodexSecurityDeepReducerInputs(context);
+ const inputs = await readDeepReductionSources(context);
const expectedScanId =
bound.scanId ??
inputs.previous?.scanId ??
@@ -104,8 +178,9 @@ export async function recordCodexSecurityDeepReduction(
inputs.previous,
);
- await saveScanDraftCheckpoint(context, reduction);
- await writeJsonAtomic(bound.resultPath, reduction);
+ const persisted = deepReductionForPersistence(reduction);
+ await saveScanDraftCheckpoint(context, persisted);
+ await writeJsonAtomic(bound.resultPath, persisted);
const documentWarning = await saveThreatModelDocument(
context,
reduction.threatModel,
diff --git a/plugins/codex-security/mcp-app/src/artifact-io.ts b/plugins/codex-security/mcp-app/src/artifact-io.ts
index a3649ca3c..1ec40ea38 100644
--- a/plugins/codex-security/mcp-app/src/artifact-io.ts
+++ b/plugins/codex-security/mcp-app/src/artifact-io.ts
@@ -7,6 +7,7 @@ import { dirname, isAbsolute, join, sep } from "node:path";
export interface DeepReducerWorkerContext {
id: string;
resultPath: string;
+ attempt?: number;
}
export interface DeepReducerContext {
diff --git a/plugins/codex-security/mcp-app/src/artifact-scan-draft.ts b/plugins/codex-security/mcp-app/src/artifact-scan-draft.ts
index 04dda1623..c40a5824c 100644
--- a/plugins/codex-security/mcp-app/src/artifact-scan-draft.ts
+++ b/plugins/codex-security/mcp-app/src/artifact-scan-draft.ts
@@ -22,6 +22,7 @@ import {
import { artifactSchemaDocuments } from "./artifact-contracts.js";
import { projectArtifactToolSchema } from "./artifact-schema-loader.js";
import { saveThreatModelDocument } from "./threat-model-document.js";
+import { requireRegularFile } from "./deep-scan/artifacts.js";
import coverageDocument from "../../schemas/coverage.schema.json" with { type: "json" };
import findingsDocument from "../../schemas/findings.schema.json" with { type: "json" };
import scanManifestDocument from "../../schemas/scan-manifest.schema.json" with { type: "json" };
@@ -128,6 +129,7 @@ interface SavedScanDraft {
attempt?: string;
}
+const pendingWorkerDrafts = new Map<string, Promise<void>>();
const findingIdentitySchema = draftDefinitions.identity;
const findingProvenanceSchema = draftDefinitions.finding.shape.provenance;
@@ -142,7 +144,11 @@ export async function recordCodexSecurityScanDraftViaWorkbench(
const parsed = parseScanDraft(input);
requireBoundScan(context, parsed, true);
const finalDeepDraft = context.mode === "deep" && parsed.complete !== false;
- if (finalDeepDraft && resolvedDeferred(parsed.coverage).length > 0) {
+ if (
+ finalDeepDraft &&
+ publication === undefined &&
+ resolvedDeferred(parsed.coverage).length > 0
+ ) {
throw new Error(
"scan draft: terminal Deep drafts cannot resolve child deferred work.",
);
@@ -327,37 +333,57 @@ export async function recordCodexSecurityWorkerScanDraft(
}
validateRetainedCoverage(parsed.coverage);
-
- const includePaths = context.includePaths ?? [context.scope || "."];
- let scoped = {
- ...parsed,
- findings: findingsWithinScope(
- parsed.findings,
+ for (const surface of parsed.coverage.surfaces as JsonObject[]) {
+ if (isObject(surface.provenance)) delete surface.provenance.scanReceiptRefs;
+ }
+ const previous = pendingWorkerDrafts.get(context.root) ?? Promise.resolve();
+ const operation = previous.then(async () => {
+ const includePaths = context.includePaths ?? [context.scope || "."];
+ let scoped = {
+ ...parsed,
+ findings: findingsWithinScope(
+ parsed.findings,
+ includePaths,
+ context.filePaths,
+ ),
+ };
+ const preserved = await preserveScanDraft(context, scoped);
+ scoped = preserved.input;
+ // Older checkpoints predate the coordinator's requested-path binding.
+ scoped.findings = findingsWithinScope(
+ scoped.findings,
includePaths,
context.filePaths,
- ),
- };
- const preserved = await preserveScanDraft(context, scoped);
- scoped = preserved.input;
- // Older checkpoints predate the coordinator's requested-path binding.
- scoped.findings = findingsWithinScope(
- scoped.findings,
- includePaths,
- context.filePaths,
- );
- const destination = await artifactDestination(
- context,
- ["result.json"],
- "worker scan draft",
+ );
+ const destination = await artifactDestination(
+ context,
+ ["result.json"],
+ "worker scan draft",
+ );
+ await replaceArtifactJson(destination, scoped);
+ return { scoped, warnings: preserved.warnings };
+ });
+ const pending = operation.then(
+ () => undefined,
+ () => undefined,
);
- await replaceArtifactJson(destination, scoped);
+ pendingWorkerDrafts.set(context.root, pending);
+ let scoped: ScanDraftInput;
+ let preservedWarnings: string[] | undefined;
+ try {
+ ({ scoped, warnings: preservedWarnings } = await operation);
+ } finally {
+ if (pendingWorkerDrafts.get(context.root) === pending)
+ pendingWorkerDrafts.delete(context.root);
+ }
+ // Convenience rendering must not delay the next worker checkpoint.
const documentWarning = await saveThreatModelDocument(
context,
scoped.threatModel,
);
const warnings = [
- ...(preserved.warnings ?? []),
+ ...(preservedWarnings ?? []),
...(documentWarning === undefined ? [] : [documentWarning]),
];
return {
@@ -452,6 +478,185 @@ export async function saveScanDraftCheckpoint(
}
}
+/** Read-only workers cannot author the checkpoints written by the draft tool. */
+export async function hasScanDraftCheckpoint(
+ context: ArtifactContext,
+ input: JsonObject,
+): Promise<boolean> {
+ const saved = await readOptionalArtifactTextWithMetadata(context, [
+ "checkpoints",
+ scanDraftCheckpointName(input),
+ ]);
+ const { handoffClaimToken: _claim, ...snapshot } = input;
+ if (saved?.contents === JSON.stringify(snapshot, null, 2) + "\n") return true;
+ const includePaths = context.includePaths ?? [context.scope || "."];
+ if (includePaths.includes(".")) return false;
+ // Older writers checkpointed inherited findings before applying the bound scope.
+ // Compare the entire projected payload, including every receipt ownership field.
+ for (const { snapshot: recorded } of await readSavedCheckpoints(
+ context,
+ "current",
+ )) {
+ const { handoffClaimToken: _recordedClaim, ...checkpoint } = recorded;
+ if (
+ isDeepStrictEqual(
+ {
+ ...checkpoint,
+ findings: findingsWithinScope(
+ checkpoint.findings as JsonObject[],
+ includePaths,
+ context.filePaths,
+ ),
+ },
+ snapshot,
+ )
+ )
+ return true;
+ }
+ return false;
+}
+
+/** Direct result files do not establish the draft tool's receipt ownership. */
+export async function retainRecordedReceiptOwnership(
+ context: ArtifactContext,
+ input: JsonObject,
+): Promise<void> {
+ if (await hasScanDraftCheckpoint(context, input)) return;
+ const coverage = input.coverage;
+ if (!isObject(coverage) || !Array.isArray(coverage.surfaces)) return;
+ for (const surface of coverage.surfaces) {
+ if (isObject(surface) && isObject(surface.provenance))
+ delete surface.provenance.scanReceiptRefs;
+ }
+}
+
+export function workerCoverageReceiptLocation(
+ context: ArtifactContext,
+ surface: JsonObject,
+ ref: string,
+): { context: ArtifactContext; components: string[] } {
+ const components = coverageReceiptComponents(ref);
+ if (context.layout !== "worker") return { context, components };
+ const workerPrefix = `artifacts/deep_discovery/workers/${basename(dirname(context.root))}/`;
+ const shared =
+ isObject(surface.provenance) &&
+ (surface.provenance.scanReceiptRefs as string[] | undefined)?.includes(ref);
+ if (
+ shared ||
+ ref.startsWith(`${workerPrefix}output/`) ||
+ ref.startsWith(`${workerPrefix}attempts/`)
+ ) {
+ return {
+ context: {
+ ...context,
+ root: dirname(dirname(dirname(dirname(dirname(context.root))))),
+ },
+ components,
+ };
+ }
+ return { context, components };
+}
+
+function sameAuthoredSurface(left: JsonObject, right: JsonObject): boolean {
+ const content = (surface: JsonObject) => {
+ const { provenance, receiptRefs, ...fields } = surface;
+ const { scanReceiptRefs: _hostRefs, ...authoredProvenance } = isObject(
+ provenance,
+ )
+ ? provenance
+ : {};
+ return {
+ ...fields,
+ receiptRefs: ((receiptRefs as string[] | undefined) ?? []).map((ref) =>
+ posix.normalize(ref),
+ ),
+ provenance: authoredProvenance,
+ };
+ };
+ return isDeepStrictEqual(content(left), content(right));
+}
+
+async function qualifyWorkerReceipts(
+ context: ArtifactContext,
+ input: ScanDraftInput,
+ retainedSurface?: (surface: JsonObject) => JsonObject | undefined,
+): Promise<void> {
+ const workerPrefix = `artifacts/deep_discovery/workers/${basename(dirname(context.root))}/`;
+ const activePrefix = `${workerPrefix}output/`;
+ const archivePrefix = `${workerPrefix}attempts/`;
+ const scanRoot = dirname(dirname(dirname(dirname(dirname(context.root)))));
+ for (const surface of input.coverage.surfaces as JsonObject[]) {
+ if (!Array.isArray(surface.receiptRefs)) continue;
+ const previous = retainedSurface?.(surface);
+ const previousRefs = (
+ (previous?.receiptRefs as string[] | undefined) ?? []
+ ).map((ref) => posix.normalize(ref));
+ const previousShared =
+ isObject(previous?.provenance) &&
+ Array.isArray(previous.provenance.scanReceiptRefs)
+ ? previous.provenance.scanReceiptRefs
+ : [];
+ const shared = new Set<string>();
+ surface.receiptRefs = await Promise.all(
+ (surface.receiptRefs as string[]).map(async (value) => {
+ const ref = posix.normalize(value);
+ if (previousRefs.includes(ref) && previousShared.includes(ref)) {
+ shared.add(ref);
+ return ref;
+ }
+ if (ref.startsWith(archivePrefix)) return ref;
+ const localRef = ref.startsWith(activePrefix)
+ ? ref.slice(activePrefix.length)
+ : ref;
+ try {
+ await requireRegularFile(
+ join(context.root, localRef),
+ context.root,
+ true,
+ );
+ return `${activePrefix}${localRef}`;
+ } catch {
+ // A newly written local receipt wins; otherwise retain its saved owner.
+ }
+ const retained = previousRefs.find((saved) => {
+ if (previousShared.includes(saved)) return false;
+ if (saved.startsWith(activePrefix)) {
+ let original = saved.slice(activePrefix.length);
+ while (original.startsWith("artifacts/archived-receipts/")) {
+ const archived = original.slice(
+ "artifacts/archived-receipts/".length,
+ );
+ original = archived.slice(archived.indexOf("/") + 1);
+ }
+ return original === localRef;
+ }
+ if (!saved.startsWith(archivePrefix)) return false;
+ const archived = saved.slice(archivePrefix.length);
+ return archived.slice(archived.indexOf("/") + 1) === localRef;
+ });
+ if (retained !== undefined) return retained;
+ if (ref.startsWith(activePrefix)) return ref;
+ try {
+ await requireRegularFile(join(scanRoot, ref), scanRoot, true);
+ shared.add(ref);
+ } catch {
+ // Unrecorded references retain the existing local-first fallback.
+ }
+ return ref;
+ }),
+ );
+ if (isObject(surface.provenance)) delete surface.provenance.scanReceiptRefs;
+ if (shared.size > 0) {
+ surface.provenance = {
+ ...(isObject(surface.provenance) ? surface.provenance : {}),
+ scanReceiptRefs: exactUnion(
+ (surface.receiptRefs as string[]).filter((ref) => shared.has(ref)),
+ ),
+ };
+ }
+ }
+}
+
async function preserveScanDraft(
context: ArtifactContext,
input: ScanDraftInput,
@@ -465,24 +670,37 @@ async function preserveScanDraft(
const warnings: string[] = [];
const currentCheckpointName = scanDraftCheckpointName(input);
const requiresClosureValidation = resolvedDeferred(input.coverage).length > 0;
- if (saveCheckpoint && !requiresClosureValidation)
+ const saveObservation = saveCheckpoint && !requiresClosureValidation;
+ if (saveObservation && context.layout !== "worker")
await saveScanDraftCheckpoint(context, input, false);
let result = structuredClone(input);
- const previousState = await readPreviousScanDraft(context);
- const previous = previousState.input;
- if (previous && previous.scanId !== input.scanId)
- throw new Error(
- "scan checkpoint: saved result belongs to a different scan.",
+ let previousState: Awaited<ReturnType<typeof readPreviousScanDraft>>;
+ let current: SavedScanDraft[];
+ let archived: SavedScanDraft[];
+ try {
+ previousState = await readPreviousScanDraft(context);
+ if (previousState.input && previousState.input.scanId !== input.scanId)
+ throw new Error(
+ "scan checkpoint: saved result belongs to a different scan.",
+ );
+ current = await readSavedCheckpoints(
+ context,
+ "current",
+ currentCheckpointName,
);
- const current: SavedScanDraft[] = await readSavedCheckpoints(
- context,
- "current",
- currentCheckpointName,
- );
- const archived =
- context.layout === "worker"
- ? await readArchivedWorkerCheckpoints(context)
- : [];
+ archived =
+ context.layout === "worker"
+ ? await readArchivedWorkerCheckpoints(context)
+ : [];
+ } catch (error) {
+ // Failed source reads must still preserve valid submitted progress.
+ if (saveObservation && context.layout === "worker") {
+ await qualifyWorkerReceipts(context, input);
+ await saveScanDraftCheckpoint(context, input, false);
+ }
+ throw error;
+ }
+ const previous = previousState.input;
if (previous) {
current.unshift({ input: previous, modifiedMs: previousState.modifiedMs });
}
@@ -491,46 +709,94 @@ async function preserveScanDraft(
right.modifiedMs - left.modifiedMs ||
Number(right.head ?? false) - Number(left.head ?? false),
);
- const currentReceipts = new Set(
- current.flatMap(({ input }) =>
- input.coverage.surfaces.flatMap((row) =>
- Array.isArray(row.receiptRefs) ? row.receiptRefs : [],
- ),
- ),
- );
const archivedReceipts = new Map<
string,
{ context: ArtifactContext; components: string[] }
>();
+ const copiedArchiveRefs = new Map<string, string>();
+ const workerPrefix = `artifacts/deep_discovery/workers/${basename(dirname(context.root))}/`;
+ const archivePrefix = `${workerPrefix}attempts/`;
+ const activePrefix = `${workerPrefix}output/`;
+ const scanRoot = dirname(dirname(dirname(dirname(dirname(context.root)))));
for (const source of archived) {
- const attempt = source.attempt!;
for (const row of source.input.coverage.surfaces as JsonObject[]) {
if (!Array.isArray(row.receiptRefs)) continue;
+ const shared = isObject(row.provenance)
+ ? ((row.provenance.scanReceiptRefs as string[] | undefined) ?? [])
+ : [];
row.receiptRefs = (row.receiptRefs as string[]).map((ref) => {
- // Mapping saved references is data-only. Validate retained paths at I/O,
- // after a corrected same-ID surface can replace an invalid old receipt.
- const components = ref
- .split("/")
- .filter((part) => part !== "" && part !== ".");
- const retained = [
- "artifacts",
- "archived-receipts",
- attempt,
- ...components,
- ].join("/");
+ if (shared.includes(ref) || !ref.startsWith(archivePrefix)) return ref;
+ // The copy retains its source attempt in its path, while the current
+ // output owns the bytes used by this publication and future retries.
+ const retained = `${activePrefix}artifacts/archived-receipts/${ref.slice(archivePrefix.length)}`;
+ copiedArchiveRefs.set(ref, retained);
archivedReceipts.set(retained, {
- context: {
- ...context,
- root: join(dirname(context.root), "attempts", attempt),
- },
- components,
+ context: { ...context, root: scanRoot },
+ components: ref
+ .split("/")
+ .filter((part) => part !== "" && part !== "."),
});
return retained;
});
}
}
+ for (const draft of [result, ...current.map((source) => source.input)]) {
+ for (const row of draft.coverage.surfaces as JsonObject[]) {
+ if (Array.isArray(row.receiptRefs))
+ row.receiptRefs = (row.receiptRefs as string[]).map(
+ (ref) => copiedArchiveRefs.get(ref) ?? ref,
+ );
+ }
+ }
+ if (context.layout === "worker") {
+ for (const source of current)
+ await qualifyWorkerReceipts(context, source.input, (surface) => surface);
+ }
+ const currentReceipts = new Set(
+ current.flatMap(({ input }) =>
+ input.coverage.surfaces.flatMap((row) =>
+ Array.isArray(row.receiptRefs) ? row.receiptRefs : [],
+ ),
+ ),
+ );
const savedSources = [...current, ...archived];
const sources = savedSources.map(({ input }) => input);
+ const ambiguousSurfaceIds = new Set<string>();
+ for (const draft of [result, ...sources]) {
+ const seen = new Set<string>();
+ for (const surface of draft.coverage.surfaces as JsonObject[]) {
+ if (typeof surface.id !== "string") continue;
+ if (seen.has(surface.id)) ambiguousSurfaceIds.add(surface.id);
+ seen.add(surface.id);
+ }
+ }
+ // A new disposition resolves fresh evidence before closeout joins prior receipts.
+ const sameSavedSurface = (left: JsonObject, right: JsonObject) =>
+ (typeof left.id === "string" &&
+ left.id === right.id &&
+ left.disposition === right.disposition &&
+ !ambiguousSurfaceIds.has(left.id)) ||
+ sameAuthoredSurface(left, right);
+ if (context.layout === "worker") {
+ // Retained observations keep saved ownership; fresh observations use this worker.
+ await qualifyWorkerReceipts(context, result, (surface) => {
+ const previous = sources
+ .flatMap((source) => source.coverage.surfaces as JsonObject[])
+ .find(
+ (saved) =>
+ (typeof surface.id === "string" &&
+ surface.id === saved.id &&
+ !ambiguousSurfaceIds.has(surface.id)) ||
+ sameAuthoredSurface(surface, saved),
+ );
+ return previous?.disposition === surface.disposition
+ ? previous
+ : undefined;
+ });
+ // Persist the resolved ownership in the observation as well as its result.
+ input = structuredClone(result);
+ if (saveObservation) await saveScanDraftCheckpoint(context, input, false);
+ }
// Older checkpoints can omit IDs already assigned in their published output.
const savedDeferred = sources.flatMap((source) => source.coverage.deferred);
const savedSurfaces = sources.flatMap((source) => source.coverage.surfaces);
@@ -794,8 +1060,38 @@ async function preserveScanDraft(
};
for (const source of sources) {
- const deferred = result.coverage.deferred;
- const dispositions = result.coverage.surfaces.filter(
+ for (const previous of source.coverage.surfaces as JsonObject[]) {
+ const current = (result.coverage.surfaces as JsonObject[]).find(
+ (surface) =>
+ coverageEntryPresent([surface], previous, ambiguousDeferredIds) &&
+ (sameSavedSurface(surface, previous) ||
+ resolvedSurfaces.has(previous)),
+ );
+ if (!current || !isObject(previous.provenance)) continue;
+ const inherited = previous.provenance.scanReceiptRefs;
+ if (!Array.isArray(inherited)) continue;
+ const refs = new Set(
+ ((current.receiptRefs as string[] | undefined) ?? []).map((ref) =>
+ posix.normalize(ref),
+ ),
+ );
+ const retained = inherited.filter(
+ (ref) => typeof ref === "string" && refs.has(ref),
+ );
+ if (retained.length === 0) continue;
+ const provenance = isObject(current.provenance) ? current.provenance : {};
+ current.provenance = {
+ ...provenance,
+ scanReceiptRefs: exactUnion(
+ Array.isArray(provenance.scanReceiptRefs)
+ ? provenance.scanReceiptRefs
+ : [],
+ retained,
+ ),
+ };
+ }
+ const deferred = result.coverage.deferred as JsonObject[];
+ const dispositions = (result.coverage.surfaces as JsonObject[]).filter(
(surface) =>
(surface.disposition === "rejected" ||
surface.disposition === "not_applicable") &&
@@ -941,11 +1237,16 @@ async function preserveScanDraft(
const resolvedIds = new Set(
[
...result.findings.map(findingCandidateId),
- ...candidateRows
+ ...dispositions
.filter((item) => !keepsGenericWork(item))
.map((item) => item.candidateId ?? item.id),
].filter((value): value is string => typeof value === "string"),
);
+ const pendingIds = new Set(
+ deferred
+ .filter((item) => !keepsGenericWork(item))
+ .map((item) => item.candidateId ?? item.id),
+ );
const previousCoverage = {
...source.coverage,
deferred: source.coverage.deferred.filter((item) => {
@@ -963,7 +1264,9 @@ async function preserveScanDraft(
return (
(keepsGenericWork(surface) ||
typeof candidateId !== "string" ||
- !resolvedIds.has(candidateId)) &&
+ (!resolvedIds.has(candidateId) &&
+ (surface.disposition === "needs_follow_up" ||
+ !pendingIds.has(candidateId)))) &&
!coverageEntryPresent(
result.coverage.surfaces as unknown[],
surface,
@@ -998,6 +1301,26 @@ async function preserveScanDraft(
}
result.coverage.deferred = normalizeDeferred(result.coverage.deferred);
result.coverage.surfaces = normalizeSurfaces(result.coverage.surfaces);
+ if (context.layout === "worker") {
+ // Older checkpoints predate the coordinator's requested-path binding.
+ result.findings = findingsWithinScope(
+ result.findings,
+ context.includePaths ?? [context.scope || "."],
+ context.filePaths,
+ );
+ }
+ const receiptLocation = (ref: string) => {
+ const surface = [result, ...sources]
+ .flatMap((draft) => draft.coverage.surfaces as JsonObject[])
+ .find(
+ (row) =>
+ isObject(row.provenance) &&
+ (row.provenance.scanReceiptRefs as string[] | undefined)?.includes(
+ ref,
+ ),
+ );
+ return workerCoverageReceiptLocation(context, surface ?? {}, ref);
+ };
// Archived attempts move their receipt files with them. Copy only evidence
// still retained by this draft into its current, independently bound output.
// Superseded surfaces must not reserve a discarded receipt's spelling.
@@ -1007,8 +1330,8 @@ async function preserveScanDraft(
if (!currentReceipts.has(ref) || !archivedReceipts.has(ref)) continue;
try {
await artifactSourcePath(
- context,
- coverageReceiptComponents(ref),
+ receiptLocation(ref).context,
+ receiptLocation(ref).components,
"saved coverage receipt",
);
archivedReceipts.delete(ref);
@@ -1031,10 +1354,10 @@ async function preserveScanDraft(
try {
components = coverageReceiptComponents(ref);
await artifactSourcePath(
- origin?.context ?? context,
+ origin?.context ?? receiptLocation(ref).context,
origin
? coverageReceiptComponents(origin.components.join("/"))
- : components,
+ : receiptLocation(ref).components,
"selected coverage receipt",
);
} catch {
@@ -1064,7 +1387,9 @@ async function preserveScanDraft(
let source: string;
try {
const components = coverageReceiptComponents(
- origin ? origin.components.join("/") : ref,
+ origin
+ ? origin.components.join("/")
+ : receiptLocation(ref).components.join("/"),
);
coverageReceiptComponents(ref);
const previous = receiptPaths.get(key);
@@ -1076,7 +1401,7 @@ async function preserveScanDraft(
{ code: "invalid_coverage_receipt_path" },
);
source = await artifactSourcePath(
- origin?.context ?? context,
+ origin?.context ?? receiptLocation(ref).context,
components,
origin ? "archived coverage receipt" : "saved coverage receipt",
);
@@ -1141,9 +1466,11 @@ async function preserveScanDraft(
for (const replacementRef of refs) {
const saved = archivedReceipts.get(replacementRef);
await artifactSourcePath(
- saved?.context ?? context,
+ saved?.context ?? receiptLocation(replacementRef).context,
coverageReceiptComponents(
- saved ? saved.components.join("/") : replacementRef,
+ saved
+ ? saved.components.join("/")
+ : receiptLocation(replacementRef).components.join("/"),
),
"replacement coverage receipt",
);
@@ -1158,6 +1485,18 @@ async function preserveScanDraft(
// Current drafts may still be incomplete; ordinary discovery admission
// diagnoses unrepaired refs as before. Archived repair keeps its errors.
if (!origin) continue;
+ if (
+ unchangedDisposition &&
+ replacementError === error &&
+ error.message.includes("unavailable")
+ ) {
+ // Missing historical evidence remains bound to its original owner.
+ // Preserve progress for recovery; discovery admission still rejects it.
+ row.receiptRefs = (row.receiptRefs as string[]).map((value) =>
+ value === ref ? origin.components.join("/") : value,
+ );
+ continue;
+ }
throw replacementError;
}
row.receiptRefs = (row.receiptRefs as string[]).filter(
@@ -1169,9 +1508,10 @@ async function preserveScanDraft(
continue;
}
if (origin) {
+ const location = receiptLocation(ref);
const destination = await artifactDestination(
- context,
- coverageReceiptComponents(ref),
+ location.context,
+ location.components,
"retained coverage receipt",
);
await fs.copyFile(source, destination);
@@ -1604,7 +1944,7 @@ async function readSavedCheckpoints(
context: ArtifactContext,
kind: "current" | "archived",
excludedCheckpoint?: string,
-): Promise<Array<SavedScanDraft & { name: string }>> {
+): Promise<Array<SavedScanDraft & { name: string; snapshot: JsonObject }>> {
let checkpointRoot = join(context.root, "checkpoints");
const checkpointRootMetadata = await lstatIfExists(checkpointRoot);
if (checkpointRootMetadata === undefined) return [];
@@ -1630,7 +1970,9 @@ async function readSavedCheckpoints(
kind === "current" ? await readCheckpointHead(context, kind) : undefined;
const checkpointHead = head?.checkpoint;
- const checkpoints: Array<SavedScanDraft & { name: string }> = [];
+ const checkpoints: Array<
+ SavedScanDraft & { name: string; snapshot: JsonObject }
+ > = [];
const entries = (
await fs.readdir(checkpointRoot, { withFileTypes: true })
).filter(
@@ -1666,6 +2008,7 @@ async function readSavedCheckpoints(
}
checkpoints.push({
input,
+ snapshot: draft,
modifiedMs:
entry.name === checkpointHead
? head!.modifiedMs
@@ -1691,9 +2034,7 @@ async function readSavedCheckpoints(
return checkpoints;
}
-function scanDraftCheckpointName(
- input: Omit<ScanDraftInput, "coverage">,
-): string {
+function scanDraftCheckpointName(input: JsonObject): string {
const { handoffClaimToken: _claim, ...snapshot } = input;
return (
createHash("sha256").update(JSON.stringify(snapshot)).digest("hex") +
@@ -1708,13 +2049,14 @@ async function readPreviousScanDraft(
const saved = await readOptionalArtifactTextWithMetadata(context, [
"result.json",
]);
+ let input: ScanDraftInput | undefined;
+ if (saved !== undefined) {
+ const stored = parseJsonObject(saved.contents, "previous scan draft");
+ await retainRecordedReceiptOwnership(context, stored);
+ input = parsePersistedScanDraft(stored);
+ }
return {
- input:
- saved === undefined
- ? undefined
- : parsePersistedScanDraft(
- parseJsonObject(saved.contents, "previous scan draft"),
- ),
+ input,
digest: draftDigest([["result.json", saved?.contents]]),
modifiedMs: saved?.modifiedMs ?? 0,
};
@@ -1822,6 +2164,9 @@ async function readArchivedWorkerCheckpoints(
}
const archived: SavedScanDraft[] = [];
+ const archivePrefix = `artifacts/deep_discovery/workers/${basename(workerRoot)}/attempts/`;
+ const activePrefix = `artifacts/deep_discovery/workers/${basename(workerRoot)}/output/`;
+ const scanRoot = dirname(dirname(dirname(dirname(canonicalWorkerRoot))));
const attempts = (
await fs.readdir(canonicalAttemptsRoot, { withFileTypes: true })
)
@@ -1873,9 +2218,9 @@ async function readArchivedWorkerCheckpoints(
);
let result: ScanDraftInput | undefined;
try {
- result = parsePersistedScanDraft(
- parseJsonObject(saved.contents, "archived scan result"),
- );
+ const stored = parseJsonObject(saved.contents, "archived scan result");
+ await retainRecordedReceiptOwnership(attemptContext, stored);
+ result = parsePersistedScanDraft(stored);
} catch {
// A failed attempt may leave an invalid replaceable result after valid checkpoints.
}
@@ -1910,9 +2255,57 @@ async function readArchivedWorkerCheckpoints(
Number(right.result) - Number(left.result) ||
right.name.localeCompare(left.name),
);
- archived.push(
- ...drafts.map((draft) => ({ ...draft, attempt: attempt.name })),
- );
+ for (const draft of drafts) {
+ // Archive moves receipts with their attempt; preserve the checkpoint bytes.
+ for (const surface of draft.input.coverage.surfaces as JsonObject[]) {
+ if (!Array.isArray(surface.receiptRefs)) continue;
+ surface.receiptRefs = await Promise.all(
+ (surface.receiptRefs as string[]).map(async (value) => {
+ const ref = posix.normalize(value);
+ if (ref.startsWith(archivePrefix)) return ref;
+ if (
+ isObject(surface.provenance) &&
+ Array.isArray(surface.provenance.scanReceiptRefs) &&
+ surface.provenance.scanReceiptRefs.includes(ref)
+ )
+ return ref;
+ if (!ref.startsWith(activePrefix)) {
+ try {
+ await requireRegularFile(
+ join(attemptRoot, ref),
+ attemptRoot,
+ true,
+ );
+ return `${archivePrefix}${attempt.name}/${ref}`;
+ } catch {
+ // An explicit shared receipt remains at the scan root.
+ }
+ try {
+ await requireRegularFile(join(scanRoot, ref), scanRoot, true);
+ const provenance = isObject(surface.provenance)
+ ? surface.provenance
+ : {};
+ surface.provenance = {
+ ...provenance,
+ scanReceiptRefs: [
+ ...new Set([
+ ...((provenance.scanReceiptRefs as
+ string[] | undefined) ?? []),
+ ref,
+ ]),
+ ],
+ };
+ return ref;
+ } catch {
+ // Worker-output receipts moved with this archived attempt.
+ }
+ }
+ return `${archivePrefix}${attempt.name}/${ref.startsWith(activePrefix) ? ref.slice(activePrefix.length) : ref}`;
+ }),
+ );
+ }
+ archived.push({ ...draft, attempt: attempt.name });
+ }
}
return archived;
}
@@ -2136,6 +2529,11 @@ function deferredEntryPresent(
isObject(previous) &&
!ambiguousGenericEntry(current, ambiguousIds) &&
!ambiguousGenericEntry(previous, ambiguousIds) &&
+ !(
+ typeof current.id === "string" &&
+ typeof previous.id === "string" &&
+ current.id !== previous.id
+ ) &&
[previous.id, previous.candidateId].some(
(id) =>
typeof id === "string" &&
@@ -2157,6 +2555,13 @@ function coverageEntryPresent(
? { question: previous.trim() }
: structuredClone(previous);
if (isObject(current) && isObject(original)) {
+ // Distinct records can describe the same candidate's separate proof gaps.
+ if (
+ typeof current.id === "string" &&
+ typeof original.id === "string" &&
+ current.id !== original.id
+ )
+ return false;
if (
ambiguousGenericEntry(current, ambiguousIds) ||
ambiguousGenericEntry(original, ambiguousIds)
@@ -2462,6 +2867,8 @@ function parsePersistedCheckpoint(
schema: z.ZodType<ScanDraftInput> = fileScanDraftInputSchema,
): ScanDraftInput {
const compatible = structuredClone(input);
+ delete compatible.previousParentCheckpoints;
+ delete compatible.canonicalParent;
delete compatible.publication;
if (isObject(compatible.scope)) {
delete compatible.scope.includePaths;
diff --git a/plugins/codex-security/mcp-app/src/deep-scan/artifact-validation.ts b/plugins/codex-security/mcp-app/src/deep-scan/artifact-validation.ts
index 7ad6f7440..2d58efaea 100644
--- a/plugins/codex-security/mcp-app/src/deep-scan/artifact-validation.ts
+++ b/plugins/codex-security/mcp-app/src/deep-scan/artifact-validation.ts
@@ -1,8 +1,9 @@
+import { createHash } from "node:crypto";
+import { dirname, posix, relative, sep } from "node:path";
import { isDeepStrictEqual } from "node:util";
-import { dirname, relative, sep } from "node:path";
import type { JsonObject } from "../types.js";
import type { AcceptedDiscovery } from "./worker-runner.js";
-import { artifactSourcePath } from "../artifact-io.js";
+import { artifactSourcePath, type ArtifactContext } from "../artifact-io.js";
import {
coverageReceiptComponents,
validateRetainedCoverage,
@@ -12,6 +13,8 @@ import { projectArtifactToolSchema } from "../artifact-schema-loader.js";
import { requireFiniteNumbers } from "../helpers/json.js";
import {
parsePersistedScanDraft,
+ retainRecordedReceiptOwnership,
+ workerCoverageReceiptLocation,
preserveFindingDetails,
saveScanDraftCheckpoint,
scanFindingIdentity,
@@ -25,13 +28,20 @@ import {
} from "./artifacts.js";
import type { DeepScanArtifacts } from "./artifacts.js";
-export type DeepReductionInput = Omit<ScanDraftInput, "coverage">;
+export type DeepReductionInput = Omit<ScanDraftInput, "coverage"> & {
+ /** Host-owned accepted coverage, omitted from the legacy persisted reducer format. */
+ sourceCoverage?: ScanDraftInput["coverage"];
+};
export const deepReductionInputSchema = projectArtifactToolSchema(
artifactSchemaDocuments["tools/deep-reducer"].$defs.reductionInput,
);
export interface DeepReductionSources {
- discoveries: { workerId: string; result: DeepReductionInput }[];
+ discoveries: {
+ workerId: string;
+ coverage?: ScanDraftInput["coverage"];
+ result: DeepReductionInput;
+ }[];
previous: DeepReductionInput | null;
}
@@ -40,6 +50,21 @@ export interface ReducerArtifactValidation {
result: DeepReductionInput;
}
+export function deepReductionToScanDraft(
+ result: DeepReductionInput,
+): ScanDraftInput {
+ const { sourceCoverage, ...draft } = structuredClone(result);
+ return { ...draft, coverage: sourceCoverage ?? unknownSourceCoverage() };
+}
+
+/** Keep v1 checkpoints readable by older binaries. */
+export function deepReductionForPersistence(
+ result: DeepReductionInput,
+): Omit<ScanDraftInput, "coverage"> {
+ const { sourceCoverage: _coverage, ...persisted } = result;
+ return persisted;
+}
+
/** Reduction combines findings; it does not complete the source workers' review. */
export function aggregateDiscoveryCoverage(
artifacts: DeepScanArtifacts,
@@ -117,10 +142,26 @@ export async function validateDiscoveryArtifacts(
artifacts: DeepScanArtifacts,
resultPath: string,
expectedScanId: string,
+ boundScope: Pick<
+ ArtifactContext,
+ "scope" | "includePaths" | "filePaths"
+ > = {},
): Promise<ScanDraftInput> {
await requireRegularFile(resultPath, artifacts.workersRoot);
+ const root = dirname(resultPath);
+ const context = {
+ root,
+ repoRoot: root,
+ layout: "worker" as const,
+ scanId: expectedScanId,
+ scope: boundScope.scope,
+ includePaths: boundScope.includePaths,
+ filePaths: boundScope.filePaths,
+ };
+ const stored = await readJsonObject(resultPath);
+ await retainRecordedReceiptOwnership(context, stored);
const result = parseStoredScanDraft(
- await readJsonObject(resultPath),
+ stored,
"Standard scan worker",
expectedScanId,
parsePersistedScanDraft,
@@ -130,7 +171,6 @@ export async function validateDiscoveryArtifacts(
"Standard scan worker wrote only a checkpoint; its audit is not complete.",
);
validateRetainedCoverage(result.coverage);
- const root = dirname(resultPath);
const receiptPaths = new Map<string, string>();
for (const row of result.coverage.surfaces as JsonObject[]) {
for (const ref of (row.receiptRefs as string[] | undefined) ?? []) {
@@ -143,9 +183,10 @@ export async function validateDiscoveryArtifacts(
`Standard scan worker coverage receipt: duplicate artifact path: ${previous} and ${normalized}`,
);
receiptPaths.set(key, normalized);
+ const location = workerCoverageReceiptLocation(context, row, ref);
await artifactSourcePath(
- { root, repoRoot: root, layout: "worker" },
- components,
+ location.context,
+ location.components,
"Standard scan worker coverage receipt",
);
}
@@ -202,11 +243,12 @@ export async function validateReducerArtifacts(
input.sources.discoveries,
input.sources.previous,
);
+ const persisted = deepReductionForPersistence(result);
await saveScanDraftCheckpoint(
{ root: artifactDir, repoRoot: artifacts.scanDir, layout: "reducer" },
- result,
+ persisted,
);
- await writeJsonAtomic(resultPath, result);
+ await writeJsonAtomic(resultPath, persisted);
} else {
validateRetainedFindings(result, [], previous);
}
@@ -246,6 +288,11 @@ export function reconcileDeepReduction(
})),
});
const result = structuredClone(input);
+ result.sourceCoverage = aggregateSourceCoverage(discoveries, previous);
+ if (result.complete === false)
+ throw new Error(
+ "Deep reduction is only a checkpoint, not a complete result.",
+ );
for (const source of [
...discoveries.map((discovery) => discovery.result),
...(previous ? [previous] : []),
@@ -301,6 +348,148 @@ export function reconcileDeepReduction(
return result;
}
+/** Keep independent reviews separate: matching labels do not resolve another pass's proof gap. */
+export function aggregateSourceCoverage(
+ discoveries: DeepReductionSources["discoveries"],
+ previous: DeepReductionInput | null,
+): ScanDraftInput["coverage"] {
+ const sources = [
+ ...(previous ? [previous.sourceCoverage ?? unknownSourceCoverage()] : []),
+ ...discoveries.map((source) => source.coverage ?? unknownSourceCoverage()),
+ ];
+ const result: ScanDraftInput["coverage"] = {
+ surfaces: [],
+ explicitExclusions: [],
+ deferred: [],
+ reviews: [],
+ };
+ for (const field of [
+ "surfaces",
+ "explicitExclusions",
+ "deferred",
+ "resolvedDeferred",
+ "openQuestions",
+ "reviews",
+ ]) {
+ const entries = sources.flatMap(
+ (source) => (source[field] as unknown[] | undefined) ?? [],
+ );
+ if (
+ entries.length ||
+ (field !== "openQuestions" && field !== "resolvedDeferred")
+ )
+ result[field] = structuredClone(entries);
+ }
+ return result;
+}
+
+function unknownSourceCoverage(): ScanDraftInput["coverage"] {
+ return {
+ surfaces: [],
+ explicitExclusions: [],
+ deferred: [],
+ };
+}
+
+/** Qualify worker-local IDs and receipt paths before combining accepted coverage. */
+export function projectDiscoveryCoverage(
+ coverage: ScanDraftInput["coverage"],
+ worker: { id: string; attempt?: number },
+ artifactPrefix: string,
+ scanReceiptRefs: ReadonlyMap<number, ReadonlySet<string>> = new Map(),
+): ScanDraftInput["coverage"] {
+ const archivePrefix = `${posix.dirname(artifactPrefix)}/attempts/`;
+ const provenance = {
+ workerId: worker.id,
+ ...(worker.attempt === undefined ? {} : { attempt: worker.attempt }),
+ };
+ const prefix = `${worker.id}-attempt-${worker.attempt ?? "unknown"}`;
+ const surfaces = coverage.surfaces;
+ const surfaceIds = new Map<unknown, string>();
+ for (const [index, surface] of surfaces.entries()) {
+ if (!surfaceIds.has(surface.id)) {
+ surfaceIds.set(surface.id, `${prefix}-surface-${index + 1}`);
+ }
+ }
+ const project = <T extends Record<string, unknown>>(item: T): T => {
+ const result = structuredClone(item);
+ const descriptions = result.provenance;
+ const projected =
+ typeof descriptions === "object" &&
+ descriptions !== null &&
+ !Array.isArray(descriptions)
+ ? (descriptions as Record<string, unknown>)
+ : {};
+ for (const key of [
+ "workerId",
+ "attempt",
+ "sourceId",
+ "candidateId",
+ "scanReceiptRefs",
+ ])
+ delete projected[key];
+ Object.assign(result, {
+ provenance: {
+ ...projected,
+ ...provenance,
+ ...(item.id === undefined ? {} : { sourceId: item.id }),
+ ...(item.candidateId === undefined
+ ? {}
+ : { candidateId: item.candidateId }),
+ },
+ });
+ if (typeof item.candidateId === "string") {
+ Object.assign(result, {
+ candidateId: `${prefix}-candidate-${createHash("sha256").update(item.candidateId).digest("hex")}`,
+ });
+ }
+ return result;
+ };
+ return {
+ reviews: [provenance],
+ surfaces: surfaces.map((surface, index) => ({
+ ...project(surface),
+ id: `${prefix}-surface-${index + 1}`,
+ receiptRefs: ((surface.receiptRefs as string[] | undefined) ?? [])
+ .map((ref) => posix.normalize(ref))
+ .map((ref) =>
+ ref.startsWith(archivePrefix) ||
+ ref.startsWith(`${artifactPrefix}/`) ||
+ scanReceiptRefs.get(index)?.has(ref)
+ ? ref
+ : `${artifactPrefix}/${ref}`,
+ ),
+ })),
+ explicitExclusions: coverage.explicitExclusions.map(project),
+ deferred: coverage.deferred.map((item, index) => ({
+ ...project(item),
+ id: `${prefix}-deferred-${index + 1}`,
+ ...(item.surfaceIds === undefined
+ ? {}
+ : {
+ surfaceIds: (item.surfaceIds as string[]).map(
+ (id) => surfaceIds.get(id) ?? id,
+ ),
+ }),
+ })),
+ ...(coverage.resolvedDeferred === undefined
+ ? {}
+ : {
+ resolvedDeferred: coverage.resolvedDeferred.map(({ id, reason }) => ({
+ id: `${prefix}-resolved-${id}`,
+ reason,
+ })),
+ }),
+ ...(coverage.openQuestions === undefined
+ ? {}
+ : {
+ openQuestions: coverage.openQuestions.map((question) =>
+ project(typeof question === "string" ? { question } : question),
+ ),
+ }),
+ };
+}
+
function findingSourceIds(finding: Record<string, unknown>): string[] {
const { sourceFindingIds, sourceFindings } = finding.provenance as {
sourceFindingIds?: string[];
diff --git a/plugins/codex-security/mcp-app/src/deep-scan/artifacts.ts b/plugins/codex-security/mcp-app/src/deep-scan/artifacts.ts
index 8f9e69dfb..572703714 100644
--- a/plugins/codex-security/mcp-app/src/deep-scan/artifacts.ts
+++ b/plugins/codex-security/mcp-app/src/deep-scan/artifacts.ts
@@ -71,6 +71,7 @@ export async function readJsonObject(
export async function requireRegularFile(
path: string,
root: string,
+ allowEmpty = false,
): Promise<void> {
const rootPath = await fs.realpath(root);
const resolvedPath = await fs.realpath(path);
@@ -91,7 +92,11 @@ export async function requireRegularFile(
fs.lstat(path),
fs.stat(path),
]);
- if (linkStat.isSymbolicLink() || !fileStat.isFile() || fileStat.size === 0) {
+ if (
+ linkStat.isSymbolicLink() ||
+ !fileStat.isFile() ||
+ (!allowEmpty && fileStat.size === 0)
+ ) {
throw new Error(`Deep Scan artifact is not a valid regular file: ${path}`);
}
}
diff --git a/plugins/codex-security/mcp-app/src/deep-scan/coordinator.ts b/plugins/codex-security/mcp-app/src/deep-scan/coordinator.ts
index 234ad798a..f5320869f 100644
--- a/plugins/codex-security/mcp-app/src/deep-scan/coordinator.ts
+++ b/plugins/codex-security/mcp-app/src/deep-scan/coordinator.ts
@@ -2,13 +2,15 @@
import { randomUUID } from "node:crypto";
import { promises as fs } from "node:fs";
import { basename, dirname, join } from "node:path";
+import { readDeepReductionSources } from "../artifact-deep-reducer.js";
import { setTimeout as delay } from "node:timers/promises";
import {
createDeepScanArtifacts,
ensureDeepScanDirectories,
} from "./artifacts.js";
import {
- aggregateDiscoveryCoverage,
+ aggregateSourceCoverage,
+ deepReductionToScanDraft,
validateDiscoveryArtifacts,
validateReducerArtifacts,
type DeepReductionInput,
@@ -366,13 +368,7 @@ export class DeepScanCoordinator {
const schedulerResult = await this.runScheduler();
if (this.canceled || this.externallyFailed) return;
const draft = schedulerResult.result
- ? {
- ...structuredClone(schedulerResult.result),
- coverage: aggregateDiscoveryCoverage(
- this.artifacts,
- schedulerResult.discoveries,
- ),
- }
+ ? deepReductionToScanDraft(schedulerResult.result)
: scanDraftInputSchema.parse({
scanId: this.state.scanId,
findings: [],
@@ -946,6 +942,7 @@ export class DeepScanCoordinator {
label: `dedup-${String(reducerSequence).padStart(4, "0")}`,
consumed,
previousReducerResultPath,
+ previousSourceCoverage: latestResult?.sourceCoverage,
}),
);
observe(reducer);
@@ -1095,12 +1092,14 @@ export class DeepScanCoordinator {
this.artifacts,
worker.resultManifestPath,
this.state.scanId,
+ this.state,
);
recovered.push({
id: worker.id,
resultPath: worker.resultManifestPath,
result,
completionSequence: worker.completionSequence,
+ attempt: worker.attempt,
});
}
return recovered.sort(compareCompletionSequence);
@@ -1156,6 +1155,34 @@ export class DeepScanCoordinator {
},
this.state.scanId,
);
+ const accepted = discoveries.filter((source) =>
+ inputs.some(
+ (input) =>
+ input.dedupWorkerId === worker.id &&
+ input.discoveryWorkerId === source.id,
+ ),
+ );
+ const sources = await readDeepReductionSources({
+ root: worker.artifactDir,
+ repoRoot: this.state.targetPath,
+ scanId: this.state.scanId,
+ layout: "reducer",
+ scope: this.state.scope,
+ includePaths: this.state.includePaths,
+ filePaths: this.state.filePaths,
+ deepReducer: {
+ scanRoot: this.artifacts.scanDir,
+ claimedWorkers: accepted.map((source) => ({
+ id: source.id,
+ resultPath: source.resultPath,
+ attempt: source.attempt,
+ })),
+ },
+ });
+ result.sourceCoverage = aggregateSourceCoverage(
+ sources.discoveries,
+ latestResult ?? null,
+ );
latestResult = result;
resultPath = worker.resultManifestPath;
}
diff --git a/plugins/codex-security/mcp-app/src/deep-scan/worker-runner.ts b/plugins/codex-security/mcp-app/src/deep-scan/worker-runner.ts
index f0c106031..8445baa15 100644
--- a/plugins/codex-security/mcp-app/src/deep-scan/worker-runner.ts
+++ b/plugins/codex-security/mcp-app/src/deep-scan/worker-runner.ts
@@ -1,7 +1,7 @@
import { promises as fs } from "node:fs";
import type { ScanDraftInput } from "../artifact-scan-draft.js";
import { dirname, join } from "node:path";
-import { getCodexSecurityDeepReducerInputs } from "../artifact-deep-reducer.js";
+import { readDeepReductionSources } from "../artifact-deep-reducer.js";
import {
validateDiscoveryArtifacts,
validateReducerArtifacts,
@@ -39,6 +39,7 @@ export interface AcceptedDiscovery {
resultPath: string;
result: ScanDraftInput;
completionSequence: number;
+ attempt?: number;
}
export type DiscoveryOutcome =
@@ -71,6 +72,7 @@ export interface ReducerRequest {
label: string;
consumed: AcceptedDiscovery[];
previousReducerResultPath?: string;
+ previousSourceCoverage?: DeepReductionInput["sourceCoverage"];
}
export interface DeepScanWorkerRunnerOptions {
@@ -159,6 +161,7 @@ export class DeepScanWorkerRunner {
artifacts,
resultPath,
run.scanId,
+ run,
);
},
});
@@ -225,6 +228,7 @@ export class DeepScanWorkerRunner {
resultPath,
result: discoveryResult!,
completionSequence: persisted.completionSequence,
+ attempt: persisted.attempt,
},
};
}
@@ -235,6 +239,7 @@ export class DeepScanWorkerRunner {
label: reducerLabel,
consumed,
previousReducerResultPath,
+ previousSourceCoverage,
} = request;
const { artifacts, run } = this.options;
const reducerRoot = join(artifacts.dedupRoot, reducerLabel);
@@ -265,18 +270,25 @@ export class DeepScanWorkerRunner {
repoRoot: run.targetPath,
scanId: run.scanId,
layout: "reducer" as const,
+ scope: run.scope,
+ includePaths: run.includePaths,
+ filePaths: run.filePaths,
deepReducer: {
scanRoot: artifacts.scanDir,
claimedWorkers: consumed.map((worker) => ({
id: worker.id,
resultPath: worker.resultPath,
+ attempt: worker.attempt,
})),
previousReducerResultPath,
},
};
// Snapshot inputs before execution: direct file output has the same
// conservation checks as the MCP writer without rereading consumed sources.
- const sources = await getCodexSecurityDeepReducerInputs(artifactContext);
+ const sources = await readDeepReductionSources(artifactContext);
+ if (sources.previous && previousSourceCoverage !== undefined) {
+ sources.previous.sourceCoverage = structuredClone(previousSourceCoverage);
+ }
let reducerValidation!: ReducerArtifactValidation;
let outcome = await this.runWorkerWithRetries({
workerId: reducerId,
diff --git a/plugins/codex-security/mcp-app/tests/deep_scan_coordinator_fixture.ts b/plugins/codex-security/mcp-app/tests/deep_scan_coordinator_fixture.ts
index bf2d32a98..6edb890be 100644
--- a/plugins/codex-security/mcp-app/tests/deep_scan_coordinator_fixture.ts
+++ b/plugins/codex-security/mcp-app/tests/deep_scan_coordinator_fixture.ts
@@ -1,3 +1,4 @@
+import type { ScanDraftInput } from "../src/artifact-scan-draft.js";
import { readJson, writeJson } from "./support/json.ts";
import type { CoordinatorOptions } from "../src/deep-scan/coordinator.js";
import type {
@@ -31,12 +32,14 @@ export const {
AsyncLock,
classifyCodexWorkerError,
startOrJoinDeepScanCoordinator,
+ recordCodexSecurityWorkerScanDraft,
} = await importModule({
stdin: {
contents: `
export * from "./registry.ts";
export { WorkbenchDeepScanStore } from "./store.ts";
export { classifyCodexWorkerError } from "./errors.ts";
+ export { recordCodexSecurityWorkerScanDraft } from "../artifact-scan-draft.ts";
`,
resolveDir: path.join(import.meta.dirname, "../src/deep-scan/"),
},
@@ -661,7 +664,7 @@ export function standardScanDraft(
scanId: string,
candidateId: string | undefined,
workerLabel: string,
-) {
+): ScanDraftInput {
return {
scanId,
threatModel: { summary: `Independent threat model for ${workerLabel}.` },
diff --git a/plugins/codex-security/mcp-app/tests/deep_scan_coverage_cases.ts b/plugins/codex-security/mcp-app/tests/deep_scan_coverage_cases.ts
new file mode 100644
index 000000000..5a1c48801
--- /dev/null
+++ b/plugins/codex-security/mcp-app/tests/deep_scan_coverage_cases.ts
@@ -0,0 +1,88 @@
+import assert from "node:assert/strict";
+import { mkdir, writeFile } from "node:fs/promises";
+import path from "node:path";
+import { readJson, writeJson } from "./support/json.ts";
+import type { ScanDraftInput } from "../src/artifact-scan-draft.js";
+import type { JsonObject } from "../src/types.js";
+import {
+ coordinatorFixture,
+ FakeExecutor,
+ runCoordinator,
+} from "./deep_scan_coordinator_fixture.ts";
+
+export async function testSuccessfulDeepCoveragePreservesWorkerReviewStatus() {
+ const { fixture, store } = await coordinatorFixture({
+ workers: 2,
+ stopAfterNoNew: 2,
+ maxDiscoveryRuns: 2,
+ });
+ const executor = new FakeExecutor();
+ const run = executor.run.bind(executor);
+ const reviewed = { label: "Reviewed query", disposition: "no_issue_found" };
+ const workerReviewed = {
+ ...reviewed,
+ receiptRefs: ["artifacts/review.md"],
+ };
+ const followUp = {
+ label: "Worker follow-up",
+ disposition: "needs_follow_up",
+ };
+ executor.run = async (request) => {
+ const outcome = await run(request);
+ if (request.kind !== "discovery") return outcome;
+ const receiptPath = path.join(
+ request.artifactContext!.root,
+ workerReviewed.receiptRefs[0],
+ );
+ await mkdir(path.dirname(receiptPath), { recursive: true });
+ await writeFile(receiptPath, "Independent source review receipt.\n");
+ const resultPath = path.join(request.artifactContext!.root, "result.json");
+ const draft = await readJson(resultPath);
+ draft.coverage = {
+ completeness:
+ request.kind === "discovery" &&
+ request.promptPath.includes("discovery-0002")
+ ? "unknown"
+ : "partial",
+ surfaces: [workerReviewed, followUp],
+ explicitExclusions: [],
+ deferred: [
+ { reason: "An independent review left this question unresolved." },
+ ],
+ };
+ await writeJson(resultPath, draft);
+ return outcome;
+ };
+ const completed: ScanDraftInput[] = [];
+ const terminal = await runCoordinator(fixture, store, executor, {
+ onComplete: async (draft) => void completed.push(structuredClone(draft)),
+ });
+ assert.equal(terminal?.status, "succeeded", terminal?.error);
+ assert.equal(completed.length, 1);
+ const coverage = completed[0].coverage;
+ assert.equal(coverage.completeness, undefined);
+ assert.equal((coverage.deferred as JsonObject[]).length, 2);
+ assert.equal((coverage.surfaces as JsonObject[]).length, 4);
+ const reviews = coverage.reviews as { workerId: string }[];
+ assert.equal(reviews.length, 2);
+ assert.deepEqual(
+ new Set(reviews.map((review) => review.workerId)),
+ new Set(
+ [...store.workers.values()]
+ .filter((worker) => worker.kind === "discovery")
+ .map((worker) => worker.id),
+ ),
+ );
+ for (const item of coverage.deferred as JsonObject[]) {
+ const provenance = item.provenance as { workerId: string; attempt: number };
+ assert.equal(provenance.attempt, 1);
+ assert.ok(store.workers.has(provenance.workerId));
+ }
+ for (const worker of store.workers.values()) {
+ if (worker.kind !== "discovery") continue;
+ const draft = await readJson(worker.resultManifestPath!);
+ assert.notEqual(draft.coverage.completeness, "complete");
+ assert.deepEqual(draft.coverage.surfaces, [workerReviewed, followUp]);
+ assert.equal(draft.coverage.deferred.length, 1);
+ }
+}
diff --git a/plugins/codex-security/mcp-app/tests/deep_scan_coverage_fixture.ts b/plugins/codex-security/mcp-app/tests/deep_scan_coverage_fixture.ts
new file mode 100644
index 000000000..44d8e4e75
--- /dev/null
+++ b/plugins/codex-security/mcp-app/tests/deep_scan_coverage_fixture.ts
@@ -0,0 +1,1236 @@
+import assert from "node:assert/strict";
+import { execFile } from "node:child_process";
+import { randomUUID } from "node:crypto";
+import { mkdir, readFile, readdir, writeFile } from "node:fs/promises";
+import path from "node:path";
+import { fileURLToPath, pathToFileURL } from "node:url";
+import { promisify } from "node:util";
+import { build } from "esbuild";
+
+import type { JsonObject } from "../src/types.js";
+import type { CodexWorkerRequest } from "../src/deep-scan/types.js";
+import { readJson } from "./support/json.ts";
+
+type FixtureRuntime = typeof import("../src/deep-scan/artifacts.js") &
+ typeof import("../src/deep-scan/coordinator.js") &
+ typeof import("../src/deep-scan/store.js") &
+ typeof import("../src/artifact-context.js") &
+ typeof import("../src/artifact-scan-draft.js") &
+ typeof import("../src/artifact-deep-reducer.js");
+
+interface CoverageFixtureOptions {
+ resume?: boolean;
+ continueAfterResume?: boolean;
+ stopAfterDraft?: boolean;
+ stopBeforeDraft?: boolean;
+ receiptRetry?: boolean;
+ firstCheckpointReceipt?: "shared" | "worker";
+ sameAttemptCloseout?: boolean;
+ extraReceiptRetry?: boolean;
+ rewriteReturnedCoverage?: boolean;
+ sameNamedNewSurface?: boolean;
+ closeRetriedSurface?: boolean;
+ receiptOwnershipRetry?: "ordinary" | "candidate" | "unrelated";
+ receiptSpelling?:
+ | "worker"
+ | "scan"
+ | "equivalent scan"
+ | "active scan"
+ | "equivalent active scan"
+ | "shared scan"
+ | "equivalent shared scan";
+ activeReceiptSpelling?: "worker" | "scan" | "equivalent scan";
+ sharedReceipt?: boolean;
+ receiptCollision?: boolean;
+ emptyReceipt?: boolean;
+ retryPending?: boolean;
+ retryCoverage?: JsonObject[];
+ retryFindings?: JsonObject[][];
+ questionRows?: (string | JsonObject)[];
+ interruptPublication?: boolean;
+}
+interface ProjectedRecord {
+ id?: string;
+ provenance: { workerId: string; attempt: number; candidateId?: string };
+}
+interface PublishedCoverage {
+ surfaces: (ProjectedRecord & {
+ id: string;
+ label: string;
+ candidateId?: string;
+ disposition: string;
+ receiptRefs: string[];
+ })[];
+ deferred: (ProjectedRecord & {
+ id: string;
+ candidateId?: string;
+ reason: string;
+ surfaceIds: string[];
+ })[];
+ resolvedDeferred?: { id: string; reason: string }[];
+ explicitExclusions: ProjectedRecord[];
+ openQuestions: (ProjectedRecord & {
+ question: string;
+ followUpPrompt?: string;
+ })[];
+ reviews: { workerId: string; attempt: number }[];
+}
+export async function readFixtureCoverage(
+ scanDir: string,
+): Promise<PublishedCoverage> {
+ return readJson(path.join(scanDir, "coverage.json"));
+}
+
+const pluginRoot = fileURLToPath(new URL("../../", import.meta.url));
+const exec = promisify(execFile);
+const bundled = await build({
+ bundle: true,
+ stdin: {
+ contents: [
+ 'export { archiveDirectory } from "./src/deep-scan/artifacts.ts";',
+ 'export { DeepScanCoordinator } from "./src/deep-scan/coordinator.ts";',
+ 'export { WorkbenchDeepScanStore } from "./src/deep-scan/store.ts";',
+ 'export { createScanArtifactContext } from "./src/artifact-context.ts";',
+ 'export { getCodexSecurityCompletedScan, recordCodexSecurityWorkerScanDraft, recordCodexSecurityScanDraftViaWorkbench } from "./src/artifact-scan-draft.ts";',
+ 'export { recordCodexSecurityDeepReduction } from "./src/artifact-deep-reducer.ts";',
+ ].join("\n"),
+ resolveDir: path.join(pluginRoot, "mcp-app"),
+ },
+ format: "esm",
+ platform: "node",
+ loader: { ".md": "text" },
+ write: false,
+});
+export async function publishCoverageFixture(
+ root: string,
+ completeness: string,
+ {
+ resume = false,
+ continueAfterResume = false,
+ stopAfterDraft = false,
+ stopBeforeDraft = false,
+ receiptRetry = false,
+ firstCheckpointReceipt,
+ sameAttemptCloseout = false,
+ extraReceiptRetry = false,
+ rewriteReturnedCoverage = false,
+ sameNamedNewSurface = false,
+ closeRetriedSurface = false,
+ receiptOwnershipRetry,
+ receiptSpelling = "worker",
+ activeReceiptSpelling = "worker",
+ sharedReceipt = false,
+ receiptCollision = false,
+ emptyReceipt = false,
+ retryPending = false,
+ retryCoverage,
+ retryFindings,
+ questionRows,
+ interruptPublication = false,
+ }: CoverageFixtureOptions = {},
+) {
+ const runtimePath = path.join(root, "fixture-runtime.mjs");
+ await writeFile(runtimePath, bundled.outputFiles![0]!.contents);
+ const {
+ archiveDirectory,
+ getCodexSecurityCompletedScan,
+ DeepScanCoordinator,
+ WorkbenchDeepScanStore,
+ createScanArtifactContext,
+ recordCodexSecurityScanDraftViaWorkbench,
+ recordCodexSecurityWorkerScanDraft,
+ recordCodexSecurityDeepReduction,
+ } = (await import(pathToFileURL(runtimePath).href)) as FixtureRuntime;
+ const targetPath = path.join(root, "target");
+ const codexHome = path.join(root, "codex-home");
+ const scanRoot = path.join(root, "scans");
+ const threadId = "coverage-fixture-owner";
+ const statuses =
+ completeness === "partial"
+ ? ["partial", "complete", "unknown"]
+ : completeness === "unknown"
+ ? ["unknown", "complete", "complete"]
+ : ["complete"];
+ await mkdir(scanRoot, { mode: 0o700 });
+ await mkdir(targetPath, { recursive: true });
+ await mkdir(path.join(codexHome, "codex-security"), { recursive: true });
+ await writeFile(path.join(targetPath, "source.py"), "# Synthetic source\n");
+ await writeFile(
+ path.join(codexHome, "codex-security", "config.toml"),
+ `[deep_scan]\nworkers = 1\nsubagents = 0\nstop_after_no_new = ${statuses.length}\nmax_discovery_runs = ${statuses.length}\n`,
+ );
+ const runWorkbench = async (args: string[]) => {
+ const workbenchPath = path.join(pluginRoot, "scripts", "workbench_db.py");
+ const command = [workbenchPath, ...args];
+ if (interruptPublication && args[0] === "fail-scan") {
+ command.unshift(
+ "-c",
+ `
+import os, runpy, sys
+sys.path.insert(0, os.path.dirname(sys.argv[1]))
+import workbench_saved_results
+
+def interrupt_publication(*args, **kwargs):
+ raise OSError("Synthetic publication interruption.")
+
+workbench_saved_results._write_prepared_scan_finalization = interrupt_publication
+sys.argv = sys.argv[1:]
+runpy.run_path(sys.argv[0], run_name="__main__")
+`,
+ );
+ }
+ const { stdout } = await exec(process.env.PYTHON || "python3", command, {
+ env: {
+ ...process.env,
+ CODEX_HOME: codexHome,
+ CODEX_SECURITY_STATE_DIR: path.join(root, "state"),
+ },
+ });
+ return JSON.parse(stdout);
+ };
+ const store = new WorkbenchDeepScanStore(runWorkbench);
+ let run = await store.begin({
+ targetPath,
+ scope: ".",
+ threadId,
+ scanRoot,
+ });
+ const context = await createScanArtifactContext(run.scanId, runWorkbench, {
+ requireRunning: true,
+ });
+ const rawSources = new Map<string, string>();
+ const writeReceiptAttempt = async (artifactDir: string) => {
+ if (retryCoverage) {
+ await mkdir(artifactDir, { recursive: true });
+ await recordCodexSecurityWorkerScanDraft(
+ {
+ root: artifactDir,
+ layout: "worker",
+ repoRoot: targetPath,
+ scanId: run.scanId,
+ },
+ {
+ scanId: run.scanId,
+ complete: false,
+ findings: retryFindings?.[0] ?? [],
+ coverage: retryCoverage[0],
+ },
+ );
+ return;
+ }
+ await mkdir(path.join(artifactDir, "artifacts"), { recursive: true });
+ await writeFile(
+ path.join(artifactDir, "artifacts", "prior.txt"),
+ emptyReceipt ? "" : "Archived receipt.\n",
+ );
+ if (receiptCollision) {
+ await mkdir(path.join(run.scanDir, "artifacts"), { recursive: true });
+ await writeFile(
+ path.join(run.scanDir, "artifacts", "prior.txt"),
+ "Synthetic unrelated parent review.\n",
+ );
+ }
+ const archivedRef = `${path.relative(run.scanDir, path.dirname(artifactDir)).split(path.sep).join("/")}/attempts/attempt-01/artifacts/prior.txt`;
+ const sharedPrior = "artifacts/01_context/false_positive_feedback.json";
+ const sharedPriorReceipt =
+ receiptSpelling === "shared scan" ||
+ receiptSpelling === "equivalent shared scan";
+ if (sharedPriorReceipt) {
+ await mkdir(path.dirname(path.join(run.scanDir, sharedPrior)), {
+ recursive: true,
+ });
+ await writeFile(
+ path.join(run.scanDir, sharedPrior),
+ emptyReceipt ? "" : "Archived receipt.\n",
+ );
+ }
+ const receiptRef = sharedPriorReceipt
+ ? sharedPrior.replace(
+ "artifacts/",
+ receiptSpelling === "equivalent shared scan"
+ ? "artifacts/./"
+ : "artifacts/",
+ )
+ : receiptSpelling === "worker"
+ ? "artifacts/prior.txt"
+ : receiptSpelling === "scan"
+ ? archivedRef
+ : receiptSpelling === "equivalent scan"
+ ? archivedRef.replace("artifacts/", "artifacts/./")
+ : `${path.relative(run.scanDir, artifactDir).split(path.sep).join("/")}/artifacts/prior.txt`.replace(
+ "artifacts/",
+ receiptSpelling === "equivalent active scan"
+ ? "artifacts/./"
+ : "artifacts/",
+ );
+ if (receiptOwnershipRetry) {
+ await recordCodexSecurityWorkerScanDraft(
+ {
+ root: artifactDir,
+ layout: "worker",
+ repoRoot: targetPath,
+ scanId: run.scanId,
+ },
+ {
+ scanId: run.scanId,
+ complete: false,
+ findings: [],
+ coverage: {
+ completeness: "partial",
+ surfaces: [
+ {
+ id: "current",
+ label: "Current review",
+ disposition:
+ receiptOwnershipRetry === "unrelated"
+ ? "no_issue_found"
+ : "needs_follow_up",
+ ...(receiptOwnershipRetry === "candidate"
+ ? { candidateId: "prior-candidate" }
+ : {}),
+ receiptRefs: [receiptRef],
+ },
+ ...(receiptOwnershipRetry === "unrelated"
+ ? [
+ {
+ id: "closed",
+ label: "Separate review",
+ disposition: "needs_follow_up",
+ receiptRefs: [receiptRef],
+ },
+ ]
+ : []),
+ ],
+ explicitExclusions: [],
+ deferred:
+ receiptOwnershipRetry === "ordinary"
+ ? []
+ : [
+ {
+ id: "prior-gap",
+ ...(receiptOwnershipRetry === "candidate"
+ ? { candidateId: "prior-candidate" }
+ : {}),
+ reason: "Verify the earlier boundary.",
+ surfaceIds: [
+ receiptOwnershipRetry === "unrelated"
+ ? "closed"
+ : "current",
+ ],
+ },
+ ],
+ },
+ },
+ );
+ return;
+ }
+ if (closeRetriedSurface) {
+ await recordCodexSecurityWorkerScanDraft(
+ {
+ root: artifactDir,
+ layout: "worker",
+ repoRoot: targetPath,
+ scanId: run.scanId,
+ },
+ {
+ scanId: run.scanId,
+ complete: false,
+ findings: [],
+ coverage: {
+ completeness: "partial",
+ surfaces: [
+ {
+ id: "current",
+ label: "Current review",
+ disposition: "needs_follow_up",
+ receiptRefs: [receiptRef],
+ },
+ ],
+ explicitExclusions: [],
+ deferred: [
+ {
+ id: "prior-gap",
+ reason: "Verify the earlier boundary.",
+ surfaceIds: ["current"],
+ },
+ ],
+ },
+ },
+ );
+ return;
+ }
+ await recordCodexSecurityWorkerScanDraft(
+ {
+ root: artifactDir,
+ layout: "worker",
+ repoRoot: targetPath,
+ scanId: run.scanId,
+ },
+ {
+ scanId: run.scanId,
+ complete: false,
+ findings: [],
+ coverage: {
+ completeness: "partial",
+ surfaces: [
+ {
+ id: "prior",
+ label: "Prior review",
+ disposition: "needs_follow_up",
+ candidateId: "shared-prior-candidate",
+ receiptRefs: [receiptRef],
+ },
+ {
+ id: "prior-second",
+ label: "Prior second surface",
+ disposition: "needs_follow_up",
+ candidateId: "shared-prior-candidate",
+ receiptRefs: [receiptRef],
+ },
+ ],
+ explicitExclusions: [],
+ deferred: [
+ {
+ id: "prior-gap",
+ candidateId: "shared-prior-candidate",
+ reason: "Verify the earlier attempt's unresolved boundary.",
+ surfaceIds: ["prior", "prior-second"],
+ },
+ {
+ id: "prior-gap-2",
+ candidateId: "shared-prior-candidate",
+ reason: "Verify the same candidate's second boundary.",
+ surfaceIds: ["prior", "prior-second"],
+ },
+ ],
+ },
+ },
+ );
+ };
+ const writeDiscovery = async (
+ artifactDir: string,
+ index: number,
+ complete = true,
+ ) => {
+ const status = statuses[index];
+ if (firstCheckpointReceipt) {
+ await mkdir(artifactDir, { recursive: true });
+ const firstRef =
+ firstCheckpointReceipt === "shared"
+ ? "artifacts/01_context/false_positive_feedback.json"
+ : "artifacts/first.txt";
+ const firstRoot =
+ firstCheckpointReceipt === "shared" ? run.scanDir : artifactDir;
+ await mkdir(path.dirname(path.join(firstRoot, firstRef)), {
+ recursive: true,
+ });
+ await writeFile(
+ path.join(firstRoot, firstRef),
+ "Synthetic first checkpoint.\n",
+ );
+ if (firstCheckpointReceipt === "worker") {
+ await mkdir(path.dirname(path.join(run.scanDir, firstRef)), {
+ recursive: true,
+ });
+ await writeFile(
+ path.join(run.scanDir, firstRef),
+ "Synthetic unrelated parent checkpoint.\n",
+ );
+ }
+ const workerContext = {
+ root: artifactDir,
+ layout: "worker" as const,
+ repoRoot: targetPath,
+ scanId: run.scanId,
+ };
+ await recordCodexSecurityWorkerScanDraft(workerContext, {
+ scanId: run.scanId,
+ complete: false,
+ findings: [],
+ coverage: {
+ completeness: "partial",
+ surfaces: [
+ {
+ id: "first",
+ label: "First checkpoint",
+ disposition: sameAttemptCloseout
+ ? "needs_follow_up"
+ : "no_issue_found",
+ receiptRefs: [firstRef],
+ provenance: { scanReceiptRefs: [firstRef] },
+ },
+ ],
+ explicitExclusions: [],
+ deferred: sameAttemptCloseout
+ ? [
+ {
+ id: "first-gap",
+ reason: "Verify the same boundary.",
+ surfaceIds: ["first"],
+ },
+ ]
+ : [],
+ },
+ });
+ for (const name of await readdir(path.join(artifactDir, "checkpoints"))) {
+ const checkpoint = path.join(artifactDir, "checkpoints", name);
+ rawSources.set(checkpoint, await readFile(checkpoint, "utf8"));
+ }
+ const nextRef = receiptCollision ? firstRef : "artifacts/later.txt";
+ await mkdir(path.dirname(path.join(artifactDir, nextRef)), {
+ recursive: true,
+ });
+ await writeFile(
+ path.join(artifactDir, nextRef),
+ "Synthetic later checkpoint.\n",
+ );
+ await recordCodexSecurityWorkerScanDraft(workerContext, {
+ scanId: run.scanId,
+ complete,
+ findings: [],
+ coverage: {
+ completeness: status,
+ surfaces: [
+ {
+ id: sameAttemptCloseout ? "first" : "later",
+ label: sameAttemptCloseout
+ ? "First checkpoint"
+ : "Later checkpoint",
+ disposition: "no_issue_found",
+ receiptRefs: [nextRef],
+ },
+ ],
+ explicitExclusions: [],
+ deferred: [],
+ ...(sameAttemptCloseout
+ ? {
+ resolvedDeferred: [
+ { id: "first-gap", reason: "The same boundary is verified." },
+ ],
+ }
+ : {}),
+ },
+ });
+ const resultPath = path.join(artifactDir, "result.json");
+ rawSources.set(resultPath, await readFile(resultPath, "utf8"));
+ return;
+ }
+ const pending = completeness === "partial" && status !== "complete";
+ const coverage: JsonObject = {
+ completeness: status,
+ reviews: [
+ { workerId: "untrusted-worker", attempt: 99, completeness: "complete" },
+ ],
+ surfaces: [
+ {
+ id: "shared-surface",
+ label: "Archive route",
+ disposition: pending ? "needs_follow_up" : "no_issue_found",
+ receiptRefs: ["artifacts/review.md"],
+ },
+ {
+ id: "shared-surface",
+ label: "Archive settings",
+ disposition: "no_issue_found",
+ receiptRefs: ["artifacts/review.md"],
+ },
+ ],
+ explicitExclusions: [
+ { pattern: "vendor/", reason: "External dependency." },
+ ],
+ deferred: pending
+ ? [
+ {
+ id: "same-id",
+ candidateId: "candidate-1",
+ reason:
+ index === 0
+ ? "Verify entry boundaries."
+ : "Verify symbolic links.",
+ paths: ["source.py"],
+ surfaceIds: ["shared-surface"],
+ },
+ ]
+ : [],
+ openQuestions:
+ questionRows ??
+ (pending ? [{ question: `Deployment question ${index + 1}.` }] : []),
+ ...(retryCoverage && index === 0
+ ? structuredClone(retryCoverage[1])
+ : {}),
+ };
+ for (const field of [
+ "surfaces",
+ "explicitExclusions",
+ "deferred",
+ "openQuestions",
+ ]) {
+ for (const item of coverage[field] as (string | JsonObject)[]) {
+ if (typeof item === "string") continue;
+ item.provenance = {
+ description: `Original ${field} context.`,
+ details: { evidence: ["source review"] },
+ workerId: "untrusted-worker",
+ attempt: 99,
+ sourceId: "untrusted-source",
+ candidateId: "untrusted-candidate",
+ };
+ }
+ }
+ await mkdir(path.join(artifactDir, "artifacts"), { recursive: true });
+ await writeFile(
+ path.join(artifactDir, "artifacts", "review.md"),
+ emptyReceipt ? "" : "Synthetic review evidence.\n",
+ );
+ if (receiptCollision) {
+ await mkdir(path.join(run.scanDir, "artifacts"), { recursive: true });
+ await writeFile(
+ path.join(run.scanDir, "artifacts", "review.md"),
+ "Synthetic unrelated parent review.\n",
+ );
+ }
+ if (
+ receiptRetry &&
+ receiptCollision &&
+ (receiptSpelling === "shared scan" ||
+ receiptSpelling === "equivalent shared scan")
+ ) {
+ const collision = path.join(
+ artifactDir,
+ "artifacts",
+ "01_context",
+ "false_positive_feedback.json",
+ );
+ await mkdir(path.dirname(collision), { recursive: true });
+ await writeFile(collision, "Synthetic unrelated retry receipt.\n");
+ }
+ const activeQualifiedRef = `${path.relative(run.scanDir, artifactDir).split(path.sep).join("/")}/artifacts/review.md`;
+ const sharedRef = "artifacts/01_context/false_positive_feedback.json";
+ if (sharedReceipt) {
+ await mkdir(path.dirname(path.join(run.scanDir, sharedRef)), {
+ recursive: true,
+ });
+ await writeFile(
+ path.join(run.scanDir, sharedRef),
+ emptyReceipt ? "" : "Synthetic review evidence.\n",
+ );
+ }
+ if (sameNamedNewSurface) {
+ const local = path.join(artifactDir, sharedRef);
+ await mkdir(path.dirname(local), { recursive: true });
+ await writeFile(local, "Synthetic review evidence.\n");
+ }
+ const closureReceiptRef = receiptCollision
+ ? receiptSpelling === "shared scan"
+ ? sharedRef
+ : "artifacts/prior.txt"
+ : "artifacts/review.md";
+ if (closeRetriedSurface) {
+ const local = path.join(artifactDir, closureReceiptRef);
+ await mkdir(path.dirname(local), { recursive: true });
+ await writeFile(local, "Synthetic review evidence.\n");
+ }
+ const activeReceiptRef = closeRetriedSurface
+ ? closureReceiptRef
+ : sameNamedNewSurface
+ ? sharedRef
+ : sharedReceipt
+ ? sharedRef
+ : activeReceiptSpelling === "worker"
+ ? "artifacts/review.md"
+ : activeReceiptSpelling === "scan"
+ ? activeQualifiedRef
+ : activeQualifiedRef.replace("artifacts/", "artifacts/./");
+ const resultPath = path.join(artifactDir, "result.json");
+ if (receiptOwnershipRetry) {
+ const newRef = receiptCollision
+ ? receiptSpelling === "shared scan"
+ ? sharedRef
+ : "artifacts/prior.txt"
+ : "artifacts/review.md";
+ const local = path.join(artifactDir, newRef);
+ await mkdir(path.dirname(local), { recursive: true });
+ await writeFile(local, "Synthetic review evidence.\n");
+ const current = {
+ id: "current",
+ label: "Current review",
+ disposition:
+ receiptOwnershipRetry === "candidate" ? "rejected" : "no_issue_found",
+ ...(receiptOwnershipRetry === "candidate"
+ ? { candidateId: "prior-candidate" }
+ : {}),
+ receiptRefs: [newRef],
+ };
+ if (receiptOwnershipRetry === "unrelated") {
+ current.receiptRefs = [
+ receiptSpelling === "shared scan"
+ ? sharedRef
+ : `${path.relative(run.scanDir, path.dirname(artifactDir)).split(path.sep).join("/")}/attempts/attempt-01/artifacts/prior.txt`,
+ ];
+ }
+ await recordCodexSecurityWorkerScanDraft(
+ {
+ root: artifactDir,
+ layout: "worker",
+ repoRoot: targetPath,
+ scanId: run.scanId,
+ },
+ {
+ scanId: run.scanId,
+ complete,
+ findings: [],
+ coverage: {
+ completeness: status,
+ surfaces: [
+ current,
+ ...(receiptOwnershipRetry === "unrelated"
+ ? [
+ {
+ id: "closed",
+ label: "Separate review",
+ disposition: "no_issue_found",
+ receiptRefs: [newRef],
+ },
+ ]
+ : []),
+ ],
+ explicitExclusions: [],
+ deferred: [],
+ ...(receiptOwnershipRetry === "unrelated"
+ ? {
+ resolvedDeferred: [
+ {
+ id: "prior-gap",
+ reason: "The separate boundary is verified.",
+ },
+ ],
+ }
+ : {}),
+ },
+ },
+ );
+ rawSources.set(resultPath, await readFile(resultPath, "utf8"));
+ return;
+ }
+ if (receiptRetry) {
+ const saved = await recordCodexSecurityWorkerScanDraft(
+ {
+ root: artifactDir,
+ repoRoot: targetPath,
+ layout: "worker",
+ scanId: run.scanId,
+ },
+ {
+ scanId: run.scanId,
+ complete,
+ findings: [],
+ coverage: {
+ completeness: retryPending ? "partial" : status,
+ surfaces: [
+ {
+ id: "current",
+ label: "Current review",
+ disposition: "no_issue_found",
+ receiptRefs: [activeReceiptRef],
+ ...(receiptCollision
+ ? { provenance: { scanReceiptRefs: [activeReceiptRef] } }
+ : {}),
+ },
+ ],
+ explicitExclusions: [],
+ ...(closeRetriedSurface
+ ? {
+ resolvedDeferred: [
+ { id: "prior-gap", reason: "The boundary is verified." },
+ ],
+ }
+ : {}),
+ deferred: retryPending
+ ? [
+ {
+ id: "prior-gap",
+ candidateId: "shared-prior-candidate",
+ reason: "The earlier boundary still needs verification.",
+ surfaceIds: ["prior", "prior-second"],
+ },
+ ]
+ : [],
+ },
+ },
+ );
+
+ if (rewriteReturnedCoverage && index === 0) {
+ await recordCodexSecurityWorkerScanDraft(
+ {
+ root: artifactDir,
+ repoRoot: targetPath,
+ layout: "worker",
+ scanId: run.scanId,
+ },
+ {
+ scanId: run.scanId,
+ complete,
+ findings: [],
+ coverage: structuredClone(saved.coverage),
+ },
+ );
+ }
+ } else {
+ await recordCodexSecurityWorkerScanDraft(
+ {
+ root: artifactDir,
+ repoRoot: targetPath,
+ layout: "worker",
+ scanId: run.scanId,
+ },
+ {
+ scanId: run.scanId,
+ complete: true,
+ findings: index === 0 ? (retryFindings?.[1] ?? []) : [],
+ coverage,
+ },
+ );
+ for (const name of await readdir(path.join(artifactDir, "checkpoints"))) {
+ const checkpointPath = path.join(artifactDir, "checkpoints", name);
+ rawSources.set(checkpointPath, await readFile(checkpointPath, "utf8"));
+ }
+ }
+ if (complete)
+ rawSources.set(resultPath, await readFile(resultPath, "utf8"));
+ };
+ if (resume) {
+ const workers = [];
+ const seeded = continueAfterResume ? statuses.slice(0, -1) : statuses;
+ for (const index of seeded.keys()) {
+ const workerRoot = path.join(
+ run.scanDir,
+ "artifacts",
+ "deep_discovery",
+ "workers",
+ `discovery-${String(index + 1).padStart(4, "0")}`,
+ );
+ const artifactDir = path.join(workerRoot, "output");
+ const worker = {
+ id: randomUUID(),
+ scanId: run.scanId,
+ kind: "discovery" as const,
+ promptPath: path.join(workerRoot, "prompt.md"),
+ artifactDir,
+ attempt: index === 0 ? (extraReceiptRetry ? 3 : 2) : 1,
+ };
+ if (receiptRetry || (retryCoverage && index === 0)) {
+ await writeReceiptAttempt(artifactDir);
+ await archiveDirectory(
+ artifactDir,
+ path.join(workerRoot, "attempts", "attempt-01"),
+ );
+ }
+ if (extraReceiptRetry && index === 0) {
+ await writeDiscovery(artifactDir, index, false);
+ await archiveDirectory(
+ artifactDir,
+ path.join(workerRoot, "attempts", "attempt-02"),
+ );
+ }
+ await writeDiscovery(artifactDir, index);
+ await writeFile(worker.promptPath, "Synthetic discovery prompt.\n");
+ for (const status of ["queued", "running", "succeeded"] as const) {
+ await store.updateWorker({
+ ...worker,
+ status,
+ ...(status === "succeeded"
+ ? { resultManifestPath: path.join(artifactDir, "result.json") }
+ : {}),
+ });
+ }
+ workers.push(worker);
+ }
+ const artifactDir = path.join(
+ run.scanDir,
+ "artifacts",
+ "deep_discovery",
+ "dedup",
+ "dedup-0001",
+ "output",
+ );
+ const promptPath = path.join(path.dirname(artifactDir), "prompt.md");
+ await mkdir(artifactDir, { recursive: true });
+ await writeFile(promptPath, "Synthetic reducer prompt.\n");
+ const id = randomUUID();
+ await store.claimDedup({
+ id,
+ scanId: run.scanId,
+ workerIds: workers.map((worker) => worker.id),
+ artifactDir,
+ promptPath,
+ });
+ const resultManifestPath = path.join(artifactDir, "result.json");
+ // Legacy accepted reducers omitted coverage entirely.
+ await writeFile(
+ resultManifestPath,
+ JSON.stringify({
+ scanId: run.scanId,
+ findings: retryFindings?.flat() ?? [],
+ }),
+ );
+ rawSources.set(
+ resultManifestPath,
+ await readFile(resultManifestPath, "utf8"),
+ );
+ await store.commitDedup({
+ id,
+ scanId: run.scanId,
+ newFindings: retryFindings ? 1 : 0,
+ resultManifestPath,
+ });
+ run = await store.get(run.scanId, threadId);
+ }
+ let discoveryCalls = 0;
+ const executor = {
+ async run(request: CodexWorkerRequest) {
+ assert.equal(
+ resume && !continueAfterResume,
+ false,
+ "accepted legacy sources should resume without new model work",
+ );
+ assert.ok(request.artifactContext);
+ const thread = request.resumeThreadId ?? randomUUID();
+ await request.onThreadStarted?.(thread);
+ if (request.kind === "discovery") {
+ discoveryCalls++;
+ const index =
+ Number(
+ path
+ .basename(path.dirname(request.artifactContext.root))
+ .split("-")
+ .at(-1),
+ ) - 1;
+ if (index === 0 && discoveryCalls === 1) {
+ if (receiptRetry || (retryCoverage && index === 0)) {
+ await writeReceiptAttempt(request.artifactContext.root);
+ }
+ return {
+ threadId: thread,
+ finalResponse: "Continue the unfinished audit.",
+ };
+ }
+ await writeDiscovery(
+ request.artifactContext.root,
+ index,
+ !(extraReceiptRetry && index === 0 && discoveryCalls === 2),
+ );
+ } else {
+ await recordCodexSecurityDeepReduction(
+ {
+ ...request.artifactContext,
+ repoRoot: targetPath,
+ scanId: run.scanId,
+ },
+ { scanId: run.scanId, findings: retryFindings?.flat() ?? [] },
+ );
+ }
+ return { threadId: thread, finalResponse: "Audit finished." };
+ },
+ };
+ run = (await store.claimCoordinator({ scanId: run.scanId, threadId })).run;
+ const coordinator = new DeepScanCoordinator({
+ run,
+ store,
+ executor,
+ pluginRoot,
+ threadId,
+ retryDelaysMs: extraReceiptRetry ? [1, 1] : [1],
+ onComplete: async (draft, signal, publication) => {
+ if (stopBeforeDraft)
+ throw new Error("Synthetic stop before parent draft.");
+ await recordCodexSecurityScanDraftViaWorkbench(
+ context,
+ draft,
+ runWorkbench,
+ signal,
+ publication,
+ );
+ },
+ });
+ coordinator.start();
+ const terminal = await coordinator.wait(undefined, 30_000);
+ assert.equal(
+ terminal?.status,
+ stopBeforeDraft ? "failed" : "succeeded",
+ terminal?.error ?? "Deep Scan did not reach the expected terminal state.",
+ );
+ assert.ok(terminal);
+ if (!retryFindings) {
+ assert.equal(
+ terminal.noNewStreak,
+ statuses.length,
+ "source coverage must not change stopping policy",
+ );
+ }
+ assert.equal(
+ discoveryCalls,
+ resume
+ ? continueAfterResume
+ ? 1
+ : 0
+ : statuses.length + (extraReceiptRetry ? 2 : 1),
+ );
+ const accepted = await store.get(run.scanId, threadId);
+ for (const worker of (accepted.persistedWorkers ?? []).filter(
+ (worker) => worker.kind === "dedup",
+ )) {
+ assert.ok(worker.resultManifestPath);
+ const result = JSON.parse(
+ await readFile(worker.resultManifestPath, "utf8"),
+ );
+ assert.equal(
+ Object.hasOwn(result, "sourceCoverage"),
+ false,
+ "v1 reducers remain readable by earlier binaries",
+ );
+ if (!rawSources.has(worker.resultManifestPath)) {
+ for (const name of await readdir(
+ path.join(worker.artifactDir, "checkpoints"),
+ )) {
+ const checkpoint = JSON.parse(
+ await readFile(
+ path.join(worker.artifactDir, "checkpoints", name),
+ "utf8",
+ ),
+ );
+ assert.equal(
+ Object.hasOwn(checkpoint, "sourceCoverage"),
+ false,
+ "v1 checkpoints remain readable by earlier binaries",
+ );
+ }
+ }
+ }
+ if (stopBeforeDraft) {
+ const recovered = await runWorkbench([
+ "recover-scan-results",
+ "--scan-id",
+ run.scanId,
+ ]);
+ assert.equal(recovered.scan.resultsRecoveryNeeded, false);
+ const coveragePath = path.join(run.scanDir, "coverage.json");
+ const published = await readFile(coveragePath, "utf8");
+ await runWorkbench(["recover-scan-results", "--scan-id", run.scanId]);
+ assert.equal(await readFile(coveragePath, "utf8"), published);
+ } else if (stopAfterDraft) {
+ const stopped = await runWorkbench([
+ "fail-scan",
+ "--scan-id",
+ run.scanId,
+ "--message",
+ "Synthetic stop after parent draft.",
+ ]);
+ assert.equal(stopped.scan.resultsRecoveryNeeded, interruptPublication);
+ const recovered = await runWorkbench([
+ "recover-scan-results",
+ "--scan-id",
+ run.scanId,
+ ]);
+ assert.equal(recovered.scan.resultsRecoveryNeeded, false);
+ } else {
+ await runWorkbench(["complete-scan", "--scan-id", run.scanId]);
+ const completed = await getCodexSecurityCompletedScan(
+ await createScanArtifactContext(run.scanId, runWorkbench),
+ { scanId: run.scanId },
+ );
+ assert.deepEqual(
+ completed.coverage,
+ JSON.parse(
+ await readFile(path.join(run.scanDir, "coverage.json"), "utf8"),
+ ),
+ );
+ }
+ for (const [file, bytes] of rawSources)
+ assert.equal(await readFile(file, "utf8"), bytes);
+ if (firstCheckpointReceipt) {
+ const coverage = await readFixtureCoverage(run.scanDir);
+ assert.deepEqual(
+ coverage.deferred.map((row) => row.id),
+ stopAfterDraft || stopBeforeDraft ? ["scan-stopped"] : [],
+ );
+ if (sameAttemptCloseout) {
+ const surface = coverage.surfaces.find(
+ (row) => row.label === "First checkpoint",
+ );
+ assert.ok(surface);
+ assert.equal(surface.disposition, "no_issue_found");
+ assert.equal(surface.receiptRefs.length, 2);
+ const receipts = await Promise.all(
+ surface.receiptRefs.map((ref) =>
+ readFile(path.join(run.scanDir, ref), "utf8"),
+ ),
+ );
+ assert.deepEqual(
+ receipts.sort(),
+ [
+ "Synthetic first checkpoint.\n",
+ "Synthetic later checkpoint.\n",
+ ].sort(),
+ );
+ assert.equal(
+ Object.hasOwn(surface.provenance ?? {}, "scanReceiptRefs"),
+ false,
+ );
+ return { scanDir: run.scanDir, threadId, terminal };
+ }
+ for (const [label, expected] of [
+ ["First checkpoint", "Synthetic first checkpoint.\n"],
+ ["Later checkpoint", "Synthetic later checkpoint.\n"],
+ ]) {
+ const surface = coverage.surfaces.find((row) => row.label === label);
+ assert.ok(surface);
+ assert.equal(surface.receiptRefs.length, 1);
+ assert.equal(
+ await readFile(path.join(run.scanDir, surface.receiptRefs[0]), "utf8"),
+ expected,
+ "Each checkpoint keeps its observed receipt owner before any attempt archive exists.",
+ );
+ assert.equal(
+ Object.hasOwn(surface.provenance ?? {}, "scanReceiptRefs"),
+ false,
+ );
+ }
+ return { scanDir: run.scanDir, threadId, terminal };
+ }
+ if (receiptOwnershipRetry) {
+ const coverage = await readFixtureCoverage(run.scanDir);
+ const current = coverage.surfaces.find(
+ (surface) => surface.label === "Current review",
+ );
+ assert.ok(current);
+ assert.equal(
+ current.disposition,
+ receiptOwnershipRetry === "candidate" ? "rejected" : "no_issue_found",
+ );
+ assert.equal(current.receiptRefs.length, 1);
+ assert.equal(
+ await readFile(path.join(run.scanDir, current.receiptRefs[0]), "utf8"),
+ receiptOwnershipRetry === "unrelated"
+ ? "Archived receipt.\n"
+ : "Synthetic review evidence.\n",
+ "Published receipt retains the source context of its actual observation.",
+ );
+ if (receiptOwnershipRetry === "unrelated") {
+ const closed = coverage.surfaces.find(
+ (surface) => surface.label === "Separate review",
+ );
+ assert.ok(closed);
+ const bytes = await Promise.all(
+ closed.receiptRefs.map((ref) =>
+ readFile(path.join(run.scanDir, ref), "utf8"),
+ ),
+ );
+ assert.deepEqual(
+ bytes.sort(),
+ ["Archived receipt.\n", "Synthetic review evidence.\n"].sort(),
+ );
+ }
+ assert.deepEqual(coverage.deferred, []);
+ return { scanDir: run.scanDir, threadId, terminal };
+ }
+ if (closeRetriedSurface) {
+ const coverage = await readFixtureCoverage(run.scanDir);
+ assert.deepEqual(coverage.deferred, []);
+ assert.equal(coverage.surfaces.length, 1);
+ const surface = coverage.surfaces[0];
+ assert.equal(surface.label, "Current review");
+ assert.equal(surface.disposition, "no_issue_found");
+ assert.equal(
+ surface.receiptRefs.length,
+ 2,
+ "Closing a saved surface keeps both source-context receipts.",
+ );
+ const receipts = await Promise.all(
+ surface.receiptRefs.map((ref) =>
+ readFile(path.join(run.scanDir, ref), "utf8"),
+ ),
+ );
+ assert.deepEqual(
+ receipts.sort(),
+ ["Archived receipt.\n", "Synthetic review evidence.\n"].sort(),
+ );
+ assert.equal(new Set(surface.receiptRefs).size, 2);
+ return { scanDir: run.scanDir, threadId, terminal };
+ }
+ if (receiptRetry) {
+ const coverage = await readFixtureCoverage(run.scanDir);
+ for (const surface of coverage.surfaces) {
+ assert.equal(
+ Object.hasOwn(surface.provenance ?? {}, "scanReceiptRefs"),
+ false,
+ "host receipt ownership is not exposed as model-authored provenance",
+ );
+ }
+ const current = coverage.surfaces.filter(
+ (surface) => surface.label === "Current review",
+ );
+ assert.equal(current.length, 1);
+ if (sharedReceipt)
+ assert.equal(
+ current[0].receiptRefs[0],
+ "artifacts/01_context/false_positive_feedback.json",
+ );
+ else
+ assert.match(
+ current[0].receiptRefs[0],
+ sameNamedNewSurface
+ ? /\/output\/artifacts\/01_context\/false_positive_feedback\.json$/
+ : /\/output\/artifacts\/review\.md$/,
+ );
+ assert.equal(
+ await readFile(path.join(run.scanDir, current[0].receiptRefs[0]), "utf8"),
+ emptyReceipt ? "" : "Synthetic review evidence.\n",
+ );
+ const prior = coverage.surfaces.filter((surface) =>
+ surface.label.startsWith("Prior "),
+ );
+ assert.equal(prior.length, 2);
+ assert.equal(prior[0].disposition, "needs_follow_up");
+ const pending = coverage.deferred.filter(
+ (item) => item.provenance?.candidateId === "shared-prior-candidate",
+ );
+ assert.equal(pending.length, 2);
+ for (const item of pending) {
+ assert.equal(item.candidateId, prior[0].candidateId);
+ assert.deepEqual(
+ item.surfaceIds,
+ prior.map((surface) => surface.id),
+ );
+ assert.ok(
+ prior.every((surface) => surface.candidateId === item.candidateId),
+ );
+ }
+ assert.deepEqual(
+ pending.map((item) => item.reason).sort(),
+ [
+ retryPending
+ ? "The earlier boundary still needs verification."
+ : "Verify the earlier attempt's unresolved boundary.",
+ "Verify the same candidate's second boundary.",
+ ].sort(),
+ );
+ if (
+ receiptSpelling === "shared scan" ||
+ receiptSpelling === "equivalent shared scan"
+ )
+ assert.equal(
+ prior[0].receiptRefs[0],
+ "artifacts/01_context/false_positive_feedback.json",
+ );
+ else
+ assert.match(
+ prior[0].receiptRefs[0],
+ /\/(?:attempts|archived-receipts)\/attempt-01\/artifacts\/prior\.txt$/,
+ );
+ assert.equal(
+ await readFile(path.join(run.scanDir, prior[0].receiptRefs[0]), "utf8"),
+ emptyReceipt ? "" : "Archived receipt.\n",
+ );
+ }
+ return { scanDir: run.scanDir, threadId, terminal };
+}
diff --git a/plugins/codex-security/mcp-app/tests/deep_scan_lifecycle_cases.ts b/plugins/codex-security/mcp-app/tests/deep_scan_lifecycle_cases.ts
index 58f7e856f..378cbd546 100644
--- a/plugins/codex-security/mcp-app/tests/deep_scan_lifecycle_cases.ts
+++ b/plugins/codex-security/mcp-app/tests/deep_scan_lifecycle_cases.ts
@@ -1374,7 +1374,11 @@ export async function testSuccessfulDeepCoverageRetainsIndependentWorkerReviewSt
const coverage = completed[0].coverage as unknown as {
surfaces: { id: string; workerId: string; receiptRefs?: string[] }[];
explicitExclusions: { workerId: string; pattern: string }[];
- deferred: { id: string; surfaceIds: string[]; workerId: string }[];
+ deferred: {
+ id: string;
+ surfaceIds: string[];
+ provenance: { workerId: string; sourceId: string; attempt: number };
+ }[];
openQuestions: { question: string; workerId: string }[];
};
@@ -1384,8 +1388,12 @@ export async function testSuccessfulDeepCoverageRetainsIndependentWorkerReviewSt
assert.equal(coverage.explicitExclusions.length, 2);
assert.equal(coverage.openQuestions.length, 2);
for (const row of coverage.deferred) {
- assert.equal(row.id, `${row.workerId}/shared-task`);
- assert.deepEqual(row.surfaceIds, [`${row.workerId}/shared-surface`]);
+ assert.equal(row.provenance.sourceId, "shared-task");
+ assert.ok(store.workers.has(row.provenance.workerId));
+ assert.equal(row.provenance.attempt, 1);
+ assert.deepEqual(row.surfaceIds, [
+ `${row.provenance.workerId}-attempt-1-surface-1`,
+ ]);
}
for (const row of coverage.surfaces) {
for (const ref of row.receiptRefs ?? []) {
diff --git a/plugins/codex-security/mcp-app/tests/deep_scan_receipt_cases.ts b/plugins/codex-security/mcp-app/tests/deep_scan_receipt_cases.ts
new file mode 100644
index 000000000..381860432
--- /dev/null
+++ b/plugins/codex-security/mcp-app/tests/deep_scan_receipt_cases.ts
@@ -0,0 +1,174 @@
+import assert from "node:assert/strict";
+import { randomUUID } from "node:crypto";
+import { once } from "node:events";
+import { mkdir, readFile, writeFile } from "node:fs/promises";
+import path from "node:path";
+import type { ScanDraftInput } from "../src/artifact-scan-draft.js";
+import type { JsonObject } from "../src/types.js";
+import type {
+ CodexWorkerRequest,
+ CodexWorkerResult,
+} from "../src/deep-scan/types.js";
+import { readJson } from "./support/json.ts";
+import {
+ coordinatorFixture,
+ eventually,
+ FakeExecutor,
+ promptContext,
+ standardScanDraft,
+ recordCodexSecurityWorkerScanDraft,
+ createCoordinator,
+} from "./deep_scan_coordinator_fixture.ts";
+
+export async function testSharedReceiptSurvivesWorkerRetry() {
+ for (const receipt of [
+ "retrying worker output",
+ "stable shared",
+ "worker local",
+ ]) {
+ const { fixture, store } = await coordinatorFixture({
+ workers: 3,
+ maxDiscoveryRuns: 3,
+ stopAfterNoNew: 10,
+ });
+ const published = Promise.withResolvers<string>();
+ const recorded = Promise.withResolvers<void>();
+ const archived = Promise.withResolvers<void>();
+ const restore = Promise.withResolvers<void>();
+ const reducer = new FakeExecutor();
+ let recordedCount = 0;
+ let retryAttempts = 0;
+ let observedArchive = false;
+ let restored = false;
+ const executor = {
+ async run(request: CodexWorkerRequest): Promise<CodexWorkerResult> {
+ if (request.kind !== "discovery") {
+ await archived.promise;
+ restore.resolve();
+ return reducer.run(request);
+ }
+ const info = await promptContext(request.promptPath);
+ await request.onThreadStarted?.(randomUUID());
+ const context = {
+ ...request.artifactContext,
+ scanId: fixture.run.scanId,
+ repoRoot: fixture.run.targetPath,
+ };
+ if (info.workerLabel === "discovery-0003") {
+ retryAttempts++;
+ if (retryAttempts === 1) {
+ // Retry archives this invalid worker result.
+ await writeFile(
+ path.join(request.workingDirectory, "result.json"),
+ "{}\n",
+ );
+ const shared = path.join(
+ fixture.run.scanDir,
+ "artifacts/retry-window-shared.txt",
+ );
+ await mkdir(path.dirname(shared), { recursive: true });
+ await writeFile(shared, "Synthetic stable review evidence.\n");
+ published.resolve(request.workingDirectory);
+ await recorded.promise;
+ // Admission validates receipt files before a later retry can archive them.
+ await eventually(
+ () =>
+ [...store.workers.values()].filter(
+ (worker) =>
+ worker.kind === "discovery" &&
+ worker.status === "succeeded",
+ ).length === 2,
+ );
+ return {};
+ }
+ await recordCodexSecurityWorkerScanDraft(
+ context,
+ standardScanDraft(fixture.run.scanId, undefined, info.workerLabel),
+ );
+ restored = true;
+ return {};
+ }
+ const retryOutput = await published.promise;
+ const draft: ScanDraftInput = standardScanDraft(
+ fixture.run.scanId,
+ undefined,
+ info.workerLabel,
+ );
+ let ref: string;
+ if (receipt === "retrying worker output") {
+ ref = path
+ .relative(
+ fixture.run.scanDir,
+ path.join(retryOutput, "result.json"),
+ )
+ .split(path.sep)
+ .join("/");
+ } else if (receipt === "stable shared") {
+ ref = "artifacts/retry-window-shared.txt";
+ } else {
+ ref = "artifacts/evidence.txt";
+ await mkdir(path.dirname(path.join(request.workingDirectory, ref)), {
+ recursive: true,
+ });
+ await writeFile(
+ path.join(request.workingDirectory, ref),
+ "Synthetic worker-local evidence.\n",
+ );
+ }
+ (draft.coverage.surfaces as JsonObject[])[0].receiptRefs = [ref];
+ await recordCodexSecurityWorkerScanDraft(context, draft);
+ const saved = await readJson(
+ path.join(request.workingDirectory, "result.json"),
+ );
+ if (receipt !== "worker local") {
+ assert.ok(
+ saved.coverage.surfaces[0].provenance.scanReceiptRefs.includes(ref),
+ );
+ }
+ if (++recordedCount === 2) recorded.resolve();
+ return {};
+ },
+ };
+ const coordinator = createCoordinator(fixture, store, executor, {
+ random: () => 0,
+ retryDelaysMs: [1],
+ clock: {
+ now: () => 1_700_000_000_000,
+ async sleep(_delay, signal) {
+ signal.throwIfAborted();
+ const output = await published.promise;
+ await assert.rejects(readFile(path.join(output, "result.json")), {
+ code: "ENOENT",
+ });
+ assert.equal(
+ await readFile(
+ path.join(
+ path.dirname(output),
+ "attempts/attempt-01/result.json",
+ ),
+ "utf8",
+ ),
+ "{}\n",
+ );
+ observedArchive = true;
+ archived.resolve();
+ await Promise.race([restore.promise, once(signal, "abort")]);
+ signal.throwIfAborted();
+ },
+ },
+ });
+ try {
+ coordinator.start();
+ const terminal = await coordinator.wait(undefined, 5_000);
+ assert.equal(observedArchive, true, receipt);
+ assert.equal(terminal?.status, "succeeded", terminal?.error);
+ assert.equal(retryAttempts, 2, receipt);
+ assert.equal(restored, true, receipt);
+ } finally {
+ restore.resolve();
+ archived.resolve();
+ recorded.resolve();
+ coordinator.cancel("test cleanup");
+ }
+ }
+}
diff --git a/plugins/codex-security/mcp-app/tests/deep_scan_startup_cases.ts b/plugins/codex-security/mcp-app/tests/deep_scan_startup_cases.ts
new file mode 100644
index 000000000..20e5f65f3
--- /dev/null
+++ b/plugins/codex-security/mcp-app/tests/deep_scan_startup_cases.ts
@@ -0,0 +1,32 @@
+import assert from "node:assert/strict";
+
+export async function testArtifactStartupTimeoutClassification(
+ runFixtureWorker: (prompt: string) => Promise<unknown>,
+) {
+ for (const prompt of [
+ "ARTIFACT_MCP_STARTUP_TIMEOUT",
+ "LEGACY_ARTIFACT_MCP_STARTUP_TIMEOUT",
+ "ARTIFACT_MCP_STARTUP_TIMEOUT_REQUEST_TIMED_OUT",
+ "ARTIFACT_MCP_STARTUP_TIMEOUT_SYNC_AUTH_WARNING",
+ "LEGACY_ARTIFACT_MCP_STARTUP_TIMEOUT_SYNC_AUTH_WARNING",
+ "ARTIFACT_MCP_STARTUP_TIMEOUT_BOTH_AUTH_WARNINGS",
+ "LEGACY_ARTIFACT_MCP_STARTUP_TIMEOUT_BOTH_AUTH_WARNINGS",
+ "ARTIFACT_MCP_STARTUP_TIMEOUT_REQUEST_TIMED_OUT_SYNC_AUTH_WARNING",
+ "ARTIFACT_MCP_STARTUP_TIMEOUT_REQUEST_TIMED_OUT_BOTH_AUTH_WARNINGS",
+ "ARTIFACT_MCP_STARTUP_TIMEOUT_WITH_MISSING_API_KEY",
+ "ARTIFACT_MCP_STARTUP_TIMEOUT_WITH_POLICY_REFUSAL",
+ "ARTIFACT_MCP_STARTUP_TIMEOUT_REQUEST_TIMED_OUT_SYNC_AUTH_WARNING_WITH_MISSING_API_KEY",
+ "ARTIFACT_MCP_STARTUP_TIMEOUT_REQUEST_TIMED_OUT_BOTH_AUTH_WARNINGS_WITH_POLICY_REFUSAL",
+ "OTHER_MCP_STARTUP_TIMEOUT",
+ "OTHER_MCP_STARTUP_TIMEOUT_REQUEST_TIMED_OUT_SYNC_AUTH_WARNING",
+ "CATALOG_AUTH_ONLY",
+ "SYNC_AUTH_ONLY",
+ ]) {
+ await assert.rejects(
+ runFixtureWorker(prompt),
+ (error: NodeJS.ErrnoException & { cause?: NodeJS.ErrnoException }) =>
+ error?.name === "Error",
+ `${prompt} should remain retryable`,
+ );
+ }
+}
diff --git a/plugins/codex-security/mcp-app/tests/fixtures/legacy-scoped-worker-receipts.json b/plugins/codex-security/mcp-app/tests/fixtures/legacy-scoped-worker-receipts.json
new file mode 100644
index 000000000..349c9e39c
--- /dev/null
+++ b/plugins/codex-security/mcp-app/tests/fixtures/legacy-scoped-worker-receipts.json
@@ -0,0 +1,134 @@
+{
+ "checkpoint": {
+ "scanId": "7fc17317-9594-49e0-b06a-d72fd7e14bba",
+ "findings": [
+ {
+ "ruleId": "cross-site-scripting.retained",
+ "identity": {
+ "anchor": "retained"
+ },
+ "title": "Unsafe request output retained",
+ "summary": "A request-controlled value reaches an HTML response.",
+ "severity": {
+ "level": "high"
+ },
+ "confidence": {
+ "level": "high",
+ "rationale": "The source establishes reachability."
+ },
+ "taxonomy": {
+ "category": "cross-site-scripting",
+ "cwe": ["CWE-79"]
+ },
+ "locations": [
+ {
+ "path": "src/route.ts",
+ "startLine": 1,
+ "endLine": 2
+ }
+ ],
+ "remediation": "Encode request-controlled values before emitting HTML.",
+ "provenance": {
+ "source": "local_plugin"
+ }
+ },
+ {
+ "ruleId": "cross-site-scripting.outside",
+ "identity": {
+ "anchor": "outside"
+ },
+ "title": "Unsafe request output outside",
+ "summary": "A request-controlled value reaches an HTML response.",
+ "severity": {
+ "level": "high"
+ },
+ "confidence": {
+ "level": "high",
+ "rationale": "The source establishes reachability."
+ },
+ "taxonomy": {
+ "category": "cross-site-scripting",
+ "cwe": ["CWE-79"]
+ },
+ "locations": [
+ {
+ "path": "other/route.ts",
+ "startLine": 1,
+ "endLine": 2
+ }
+ ],
+ "remediation": "Encode request-controlled values before emitting HTML.",
+ "provenance": {
+ "source": "local_plugin"
+ }
+ }
+ ],
+ "coverage": {
+ "completeness": "complete",
+ "surfaces": [
+ {
+ "id": "reviewed-route",
+ "label": "Reviewed route",
+ "disposition": "no_issue_found",
+ "receiptRefs": ["artifacts/evidence.txt"],
+ "provenance": {
+ "scanReceiptRefs": ["artifacts/evidence.txt"]
+ }
+ }
+ ],
+ "explicitExclusions": [],
+ "deferred": []
+ }
+ },
+ "result": {
+ "scanId": "7fc17317-9594-49e0-b06a-d72fd7e14bba",
+ "findings": [
+ {
+ "ruleId": "cross-site-scripting.retained",
+ "identity": {
+ "anchor": "retained"
+ },
+ "title": "Unsafe request output retained",
+ "summary": "A request-controlled value reaches an HTML response.",
+ "severity": {
+ "level": "high"
+ },
+ "confidence": {
+ "level": "high",
+ "rationale": "The source establishes reachability."
+ },
+ "taxonomy": {
+ "category": "cross-site-scripting",
+ "cwe": ["CWE-79"]
+ },
+ "locations": [
+ {
+ "path": "src/route.ts",
+ "startLine": 1,
+ "endLine": 2
+ }
+ ],
+ "remediation": "Encode request-controlled values before emitting HTML.",
+ "provenance": {
+ "source": "local_plugin"
+ }
+ }
+ ],
+ "coverage": {
+ "completeness": "complete",
+ "surfaces": [
+ {
+ "id": "reviewed-route",
+ "label": "Reviewed route",
+ "disposition": "no_issue_found",
+ "receiptRefs": ["artifacts/evidence.txt"],
+ "provenance": {
+ "scanReceiptRefs": ["artifacts/evidence.txt"]
+ }
+ }
+ ],
+ "explicitExclusions": [],
+ "deferred": []
+ }
+ }
+}
diff --git a/plugins/codex-security/mcp-app/tests/test_artifact_scan_draft.ts b/plugins/codex-security/mcp-app/tests/test_artifact_scan_draft.ts
index 78f938de8..4331d55ed 100644
--- a/plugins/codex-security/mcp-app/tests/test_artifact_scan_draft.ts
+++ b/plugins/codex-security/mcp-app/tests/test_artifact_scan_draft.ts
@@ -1893,13 +1893,32 @@ try {
const partialDeferredFinding = {
summary: "Partial evidence captured before validation.",
};
+ const partialDeferredDraft = {
+ ...workerInput,
+ complete: false,
+ findings: [],
+ coverage: {
+ ...coverage,
+ completeness: "partial",
+ surfaces: [],
+ deferred: [
+ {
+ id: "first-gap",
+ candidateId: finding.provenance.candidateId,
+ reason: "Validation is pending.",
+ finding: partialDeferredFinding,
+ },
+ {
+ id: "second-gap",
+ candidateId: finding.provenance.candidateId,
+ reason: "The second boundary also needs validation.",
+ },
+ ],
+ },
+ };
await recordCodexSecurityWorkerScanDraft(
partialDeferredContext,
- partialWorkerInput({
- candidateId: finding.provenance.candidateId,
- reason: "Validation is pending.",
- finding: partialDeferredFinding,
- }),
+ partialDeferredDraft,
);
await recordCodexSecurityWorkerScanDraft(partialDeferredContext, workerInput);
const resolvedPartialDeferred = await readJson(
@@ -1910,6 +1929,31 @@ try {
resolvedPartialDeferred.findings[0].provenance.previousFindings,
[partialDeferredFinding],
);
+ assert.deepEqual(resolvedPartialDeferred.coverage.deferred, []);
+
+ for (const disposition of ["rejected", "not_applicable"]) {
+ const dispositionRoot = path.join(root, `${disposition}-deferred-worker`);
+ await mkdir(dispositionRoot);
+ const dispositionContext = { ...workerContext, root: dispositionRoot };
+ await recordCodexSecurityWorkerScanDraft(
+ dispositionContext,
+ partialDeferredDraft,
+ );
+ await recordCodexSecurityWorkerScanDraft(dispositionContext, {
+ ...workerInput,
+ findings: [],
+ coverage: {
+ ...coverage,
+ surfaces: [{ ...rejection, disposition }],
+ },
+ });
+ const resolved = JSON.parse(
+ await readFile(path.join(dispositionRoot, "result.json"), "utf8"),
+ );
+ assert.deepEqual(resolved.coverage.deferred, []);
+ assert.equal(resolved.coverage.completeness, undefined);
+ assert.equal(resolved.coverage.surfaces[0].disposition, disposition);
+ }
const recorded = await recordCodexSecurityScanDraft(context, input);
assert.deepEqual(recorded, {
diff --git a/plugins/codex-security/mcp-app/tests/test_artifact_scan_draft_recovery.ts b/plugins/codex-security/mcp-app/tests/test_artifact_scan_draft_recovery.ts
index 8d07964d7..964776dc3 100644
--- a/plugins/codex-security/mcp-app/tests/test_artifact_scan_draft_recovery.ts
+++ b/plugins/codex-security/mcp-app/tests/test_artifact_scan_draft_recovery.ts
@@ -650,7 +650,7 @@ from pathlib import Path
sys.path.insert(0,sys.argv[1])
from workbench_saved_results import merge_saved_results
root,output=Path(sys.argv[2]),Path(sys.argv[3])
-worker={"id":"worker","kind":"discovery","artifact_dir":str(output),"result_manifest_path":None,"attempt":1}
+worker={"id":"worker","kind":"discovery","artifact_dir":str(output),"result_manifest_path":None,"attempt":1,"status":"failed","merged_output_path":None}
binding={"status":"interrupted","allowedTargetKinds":["git_revision"],"target":{"kind":"git_revision","repository":"synthetic","revision":"head"},"scope":{"includePaths":["."],"excludePaths":[]},"coverageMode":"repository"}
first=merge_saved_results(root,sys.argv[4],binding,[worker],[],stopped=True,reason="interrupted")
replay=merge_saved_results(root,sys.argv[4],binding,[worker],[],stopped=True,reason="interrupted",frozen_source_digests=first[0]["scan"]["preservedSources"])
@@ -915,7 +915,7 @@ for (const layout of ["standard", "diff", "worker"] as const) {
}
});
- test(`${layout}: a legacy ID-less checkpoint retains one closable task`, async (t) => {
+ test(`${layout}: a legacy ID-less checkpoint retains one closable task through host snapshots`, async (t) => {
const f = await fixture(t, layout);
await saveScanDraftCheckpoint(
f.context,
@@ -923,7 +923,24 @@ for (const layout of ["standard", "diff", "worker"] as const) {
false,
);
let id;
- for (let attempt = 0; attempt < 3; attempt += 1) {
+ for (const canonicalParent of [undefined, false, true]) {
+ if (canonicalParent !== undefined) {
+ await assert.rejects(
+ f.write({ ...f.draft({}, true), canonicalParent }),
+ /canonicalParent/,
+ );
+ const headPath = path.join(f.root, "checkpoint-head.json");
+ const head = await readJson(headPath);
+ const snapshot = {
+ ...(await readJson(f.root, "checkpoints", head.checkpoint)),
+ canonicalParent,
+ previousParentCheckpoints: [`checkpoints/${head.checkpoint}`],
+ };
+ const contents = JSON.stringify(snapshot);
+ const checkpoint = `${hash("sha256", contents)}.json`;
+ await writeFile(path.join(f.root, "checkpoints", checkpoint), contents);
+ await writeJson(headPath, { checkpoint });
+ }
const result = await f.write(f.draft({}, true));
assert.equal(result.coverage.deferred.length, 1);
id ??= result.coverage.deferred[0].id;
diff --git a/plugins/codex-security/mcp-app/tests/test_artifact_scan_identity.ts b/plugins/codex-security/mcp-app/tests/test_artifact_scan_identity.ts
index 80f4df041..b21d174e9 100644
--- a/plugins/codex-security/mcp-app/tests/test_artifact_scan_identity.ts
+++ b/plugins/codex-security/mcp-app/tests/test_artifact_scan_identity.ts
@@ -130,6 +130,7 @@ for (const variant of [
{
id: "worker",
kind: "discovery",
+ status: "succeeded",
artifact_dir: interruptedRoot,
result_manifest_path: path.join(interruptedRoot, "result.json"),
attempt: 1,
@@ -272,6 +273,7 @@ for (const observations of [
{
id: "worker",
kind: "discovery",
+ status: "canceled",
artifact_dir: interruptedRoot,
result_manifest_path: null,
attempt: 1,
@@ -586,6 +588,7 @@ for (const { variant, expectedCount, observation } of terminalDraftCases) {
{
id: "worker",
kind: "discovery",
+ status: "canceled",
artifact_dir: interrupted.root,
result_manifest_path: null,
attempt: 1,
@@ -781,6 +784,7 @@ for (const variant of [
workers.push({
id,
kind: "discovery",
+ status: "canceled",
artifact_dir: root,
result_manifest_path: null,
attempt: 1,
@@ -866,6 +870,7 @@ for (const [kind, metadata, identifier] of [
{
id: "reviewer",
kind: "discovery",
+ status: "canceled",
artifact_dir: recoveryRoot,
result_manifest_path: null,
attempt: 1,
@@ -974,6 +979,7 @@ for (const variant of [
{
id: "reviewer",
kind: "discovery",
+ status: "canceled",
artifact_dir: rwroot,
result_manifest_path: null,
attempt: 1,
@@ -1035,6 +1041,7 @@ for (const layout of ["standard", "diff", "worker"] as const) {
workers.push({
id: "reviewer",
kind: "discovery",
+ status: "canceled",
artifact_dir: recoveredSource.root,
result_manifest_path: null,
attempt: 1,
@@ -1259,6 +1266,7 @@ for (const field of ["reportId", "ledgerRowId"]) {
{
id: "reviewer",
kind: "discovery",
+ status: "canceled",
artifact_dir: recoveredRoot,
result_manifest_path: null,
attempt: 1,
@@ -1408,6 +1416,7 @@ test("deep: refined candidate retains independent cross-location report", async
{
id: "reviewer",
kind: "discovery",
+ status: "canceled",
artifact_dir: recoveredRoot,
result_manifest_path: null,
attempt: 1,
@@ -1574,6 +1583,7 @@ for (const authored of [false, true]) {
workers.push({
id,
kind: "discovery",
+ status: "canceled",
artifact_dir: workerRoot,
result_manifest_path: null,
attempt: 1,
@@ -1658,6 +1668,7 @@ for (const order of ["earlier-first", "later-first"]) {
{
id: "reviewer",
kind: "discovery",
+ status: "canceled",
artifact_dir: recoveredRoot,
result_manifest_path: null,
attempt: 1,
@@ -1763,6 +1774,7 @@ for (const sameTitle of [false, true]) {
workers.push({
id,
kind: "discovery",
+ status: "canceled",
artifact_dir: workerRoot,
result_manifest_path: null,
attempt: 1,
@@ -1809,6 +1821,7 @@ for (const authored of [false, true]) {
{
id: "worker-a",
kind: "discovery",
+ status: "canceled",
artifact_dir: workerRoot,
result_manifest_path: null,
attempt: 1,
@@ -1823,6 +1836,7 @@ for (const authored of [false, true]) {
const savedWorker = (root: string) => ({
id: "reviewer",
kind: "discovery",
+ status: "canceled",
artifact_dir: root,
result_manifest_path: null,
attempt: 1,
@@ -2457,6 +2471,7 @@ for (const [field, value] of Object.entries({
workers.push({
id,
kind: "discovery",
+ status: "canceled",
artifact_dir: workerRoot,
result_manifest_path: null,
attempt: 1,
@@ -2658,6 +2673,7 @@ for (const field of ["description", "source", "version"]) {
{
id: "reviewer",
kind: "discovery",
+ status: "canceled",
artifact_dir: workerRoot,
result_manifest_path: null,
attempt: 1,
@@ -2705,6 +2721,7 @@ for (const metadata of ["extensions", "provenance"] as const) {
{
id: "reviewer",
kind: "discovery",
+ status: "canceled",
artifact_dir: workerRoot,
result_manifest_path: null,
attempt: 1,
@@ -2790,6 +2807,7 @@ for (const field of ["description", "source", "version"]) {
{
id: "reviewer",
kind: "discovery",
+ status: "canceled",
artifact_dir: recoveredRoot,
result_manifest_path: null,
attempt: 1,
@@ -2906,6 +2924,7 @@ for (const sameLocation of [false, true]) {
{
id: "reviewer",
kind: "discovery",
+ status: "canceled",
artifact_dir: workerRoot,
result_manifest_path: null,
attempt: 1,
@@ -2937,6 +2956,7 @@ for (const layout of ["standard", "diff", "deep", "worker"] as const) {
workers.push({
id: "reviewer",
kind: "discovery",
+ status: "canceled",
artifact_dir: recoveredRoot,
result_manifest_path: null,
attempt: 1,
@@ -3055,6 +3075,7 @@ for (const layout of ["standard", "diff", "deep", "worker"] as const) {
workers.push({
id: "reviewer",
kind: "discovery",
+ status: "canceled",
artifact_dir: recoveredRoot,
result_manifest_path: null,
attempt: 1,
@@ -3380,6 +3401,7 @@ for (const layout of ["standard", "diff", "deep", "worker"] as const) {
workers.push({
id: "reviewer",
kind: "discovery",
+ status: "canceled",
artifact_dir: recoveredRoot,
result_manifest_path: null,
attempt: 1,
@@ -3911,6 +3933,7 @@ for (const missing of [false, true]) {
{
id: "reviewer",
kind: "discovery",
+ status: "canceled",
artifact_dir: workerRoot,
result_manifest_path: null,
attempt: 1,
diff --git a/plugins/codex-security/mcp-app/tests/test_artifact_scan_publication_recovery.ts b/plugins/codex-security/mcp-app/tests/test_artifact_scan_publication_recovery.ts
index f0401e8da..6cad19806 100644
--- a/plugins/codex-security/mcp-app/tests/test_artifact_scan_publication_recovery.ts
+++ b/plugins/codex-security/mcp-app/tests/test_artifact_scan_publication_recovery.ts
@@ -79,6 +79,7 @@ for (const authored of [false, true]) {
{
id: "reviewer",
kind: "discovery",
+ status: "succeeded",
artifact_dir: workerRoot,
result_manifest_path: path.join(workerRoot, "result.json"),
attempt: 1,
@@ -336,6 +337,7 @@ test("a later worker attempt survives the terminal parent's clock tie", async (t
{
id: "reviewer",
kind: "discovery",
+ status: "succeeded",
artifact_dir: workerRoot,
result_manifest_path: path.join(workerRoot, "result.json"),
attempt: 2,
@@ -772,6 +774,7 @@ for (const [metadata, preservedIdentity] of Object.entries({
workers.push({
id: "reviewer",
kind: "discovery",
+ status: "succeeded",
artifact_dir: workerRoot,
result_manifest_path: path.join(workerRoot, "result.json"),
attempt: 1,
diff --git a/plugins/codex-security/mcp-app/tests/test_coverage_provenance.ts b/plugins/codex-security/mcp-app/tests/test_coverage_provenance.ts
new file mode 100644
index 000000000..aad77b53a
--- /dev/null
+++ b/plugins/codex-security/mcp-app/tests/test_coverage_provenance.ts
@@ -0,0 +1,170 @@
+import assert from "node:assert/strict";
+import { mkdtemp, readFile, rm } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import path from "node:path";
+import { test } from "node:test";
+import {
+ publishCoverageFixture,
+ readFixtureCoverage,
+} from "./deep_scan_coverage_fixture.ts";
+
+for (const resume of [false, true]) {
+ for (const outcome of ["completion", "recovery", "no parent"]) {
+ test(`resolved generic tasks survive ${resume ? "resumed" : "live"} ${outcome}`, async () => {
+ const root = await mkdtemp(path.join(tmpdir(), "resolved-coverage-"));
+ try {
+ const { scanDir } = await publishCoverageFixture(root, "complete", {
+ resume,
+ firstCheckpointReceipt: "worker",
+ sameAttemptCloseout: true,
+ stopAfterDraft: outcome === "recovery",
+ stopBeforeDraft: outcome === "no parent",
+ });
+ const coverage = await readFixtureCoverage(scanDir);
+ assert.deepEqual(
+ coverage.resolvedDeferred,
+ coverage.reviews.map(({ workerId, attempt }) => ({
+ id: `${workerId}-attempt-${attempt}-resolved-first-gap`,
+ reason: "The same boundary is verified.",
+ })),
+ );
+ } finally {
+ await rm(root, { recursive: true, force: true });
+ }
+ });
+ }
+}
+
+for (const resume of [false, true]) {
+ for (const outcome of ["completion", "recovery", "no parent"]) {
+ test(`coverage provenance survives ${resume ? "reconstructed" : "live"} reduction and ${outcome}`, async () => {
+ const root = await mkdtemp(path.join(tmpdir(), "coverage-provenance-"));
+ try {
+ const { scanDir } = await publishCoverageFixture(root, "partial", {
+ resume,
+ stopAfterDraft: outcome === "recovery",
+ stopBeforeDraft: outcome === "no parent",
+ });
+ const coverage = await readFixtureCoverage(scanDir);
+ assert.equal("completeness" in coverage, false);
+ assert.equal(coverage.reviews.length, 3);
+ assert.deepEqual(
+ coverage.surfaces.map((surface) => surface.label),
+ Array(3).fill(["Archive route", "Archive settings"]).flat(),
+ );
+ assert.deepEqual(
+ coverage.deferred
+ .filter((item) => item.id !== "scan-stopped")
+ .map((item) => item.reason),
+ ["Verify entry boundaries.", "Verify symbolic links."],
+ );
+ for (const item of coverage.deferred.filter(
+ (item) => item.id !== "scan-stopped",
+ )) {
+ const linked = coverage.surfaces.find(
+ (surface) => surface.id === item.surfaceIds[0],
+ );
+ assert.ok(linked);
+ assert.equal(linked.label, "Archive route");
+ assert.equal(linked.provenance.workerId, item.provenance.workerId);
+ assert.equal(linked.provenance.attempt, item.provenance.attempt);
+ }
+ for (const surface of coverage.surfaces) {
+ assert.equal(
+ await readFile(path.join(scanDir, surface.receiptRefs[0]), "utf8"),
+ "Synthetic review evidence.\n",
+ );
+ }
+ for (const field of [
+ "surfaces",
+ "explicitExclusions",
+ "deferred",
+ "openQuestions",
+ ] as const) {
+ const records = coverage[field].filter(
+ (item) => item.id !== "scan-stopped",
+ );
+ assert.ok(records.length > 0, field);
+ for (const [index, item] of records.entries()) {
+ const review = coverage.reviews.find(
+ (review) => review.workerId === item.provenance.workerId,
+ );
+ assert.ok(review, "ownership comes from the accepted worker");
+ assert.deepEqual(item.provenance, {
+ description: `Original ${field} context.`,
+ details: { evidence: ["source review"] },
+ workerId: review.workerId,
+ attempt: review.attempt,
+ ...(field === "surfaces"
+ ? {
+ sourceId:
+ index % 2 === 0 ? "shared-surface" : "shared-surface-2",
+ }
+ : {}),
+ ...(field === "deferred"
+ ? { sourceId: "same-id", candidateId: "candidate-1" }
+ : {}),
+ });
+ }
+ }
+ } finally {
+ await rm(root, { recursive: true, force: true });
+ }
+ });
+ }
+}
+
+for (const resume of [false, true]) {
+ test(`legacy coverage preserves exclusions through ${resume ? "resumed" : "live"} subsequent review`, async () => {
+ const root = await mkdtemp(path.join(tmpdir(), "unknown-coverage-"));
+ try {
+ const { scanDir } = await publishCoverageFixture(root, "unknown", {
+ resume,
+ continueAfterResume: resume,
+ });
+ const coverage = await readFixtureCoverage(scanDir);
+ assert.equal("completeness" in coverage, false);
+ assert.equal(coverage.reviews.length, 3);
+ assert.equal(coverage.explicitExclusions.length, 3);
+ } finally {
+ await rm(root, { recursive: true, force: true });
+ }
+ });
+}
+
+for (const outcome of ["completion", "recovery", "frozen recovery"]) {
+ test(`open questions are retained once through ${outcome}`, async () => {
+ const root = await mkdtemp(path.join(tmpdir(), "question-coverage-"));
+ try {
+ const { scanDir } = await publishCoverageFixture(root, "complete", {
+ stopAfterDraft: outcome !== "completion",
+ interruptPublication: outcome === "frozen recovery",
+ questionRows: [
+ " Which deployment controls apply? ",
+ {
+ question: " Which runtime settings apply? ",
+ followUpPrompt: " Check the deployment settings. ",
+ },
+ ],
+ });
+ const coverage = await readFixtureCoverage(scanDir);
+ assert.deepEqual(
+ coverage.openQuestions.map((item) => item.question),
+ ["Which deployment controls apply?", "Which runtime settings apply?"],
+ );
+ assert.equal(
+ coverage.openQuestions[1].followUpPrompt,
+ " Check the deployment settings. ",
+ );
+ for (const question of coverage.openQuestions) {
+ assert.equal(
+ question.provenance.workerId,
+ coverage.reviews[0].workerId,
+ );
+ assert.equal(question.provenance.attempt, coverage.reviews[0].attempt);
+ }
+ } finally {
+ await rm(root, { recursive: true, force: true });
+ }
+ });
+}
diff --git a/plugins/codex-security/mcp-app/tests/test_deep_scan_artifact_validation.ts b/plugins/codex-security/mcp-app/tests/test_deep_scan_artifact_validation.ts
index 1a13b09cb..eeb227ec3 100644
--- a/plugins/codex-security/mcp-app/tests/test_deep_scan_artifact_validation.ts
+++ b/plugins/codex-security/mcp-app/tests/test_deep_scan_artifact_validation.ts
@@ -15,12 +15,15 @@ import {
import path from "node:path";
import { importSource } from "./import-module.ts";
import type { ScanDraftInput } from "../src/artifact-scan-draft.ts";
+import type { DeepReductionInput } from "../src/deep-scan/artifact-validation.ts";
import type { JsonObject } from "../src/types.ts";
const {
validateDiscoveryArtifacts,
validateReducerArtifacts,
aggregateDiscoveryCoverage,
+ aggregateSourceCoverage,
+ projectDiscoveryCoverage,
} = await importSource(
path.join(import.meta.dirname, "../src/deep-scan/artifact-validation.ts"),
);
@@ -43,6 +46,7 @@ try {
await testCoveragePublicationParity(root);
await testWorkerPublicationBeforeCheckpoint(root);
await testLegacyDeferredPublication(root);
+ testResolvedDeferredPublication();
await testMissingReceiptRepair(root);
for (const reverse of [false, true])
await testMissingReceiptRepair(root, { shared: true, reverse });
@@ -74,6 +78,45 @@ try {
console.log("deep scan artifact validation tests passed");
+function testResolvedDeferredPublication() {
+ const source = {
+ surfaces: [],
+ explicitExclusions: [],
+ deferred: [{ id: "pending", reason: "A separate task remains." }],
+ resolvedDeferred: [{ id: "1", reason: "The original review finished." }],
+ };
+ let previous: DeepReductionInput | null = null;
+ for (const worker of [
+ { id: "first", attempt: 1 },
+ { id: "second", attempt: 1 },
+ { id: "first", attempt: 2 },
+ ]) {
+ const coverage = projectDiscoveryCoverage(source, worker, "worker/output");
+ const result = { scanId, findings: [] };
+ previous = {
+ ...result,
+ sourceCoverage: aggregateSourceCoverage(
+ [{ workerId: worker.id, coverage, result }],
+ previous,
+ ),
+ };
+ }
+ const parsed: ScanDraftInput = parseScanDraft({
+ ...draft([]),
+ coverage: previous!.sourceCoverage,
+ });
+ const closures = parsed.coverage.resolvedDeferred ?? [];
+ assert.equal(new Set(closures.map((row) => row.id)).size, 3);
+ assert.deepEqual(
+ closures.map((row) => row.reason),
+ Array(3).fill("The original review finished."),
+ );
+ assert.equal(parsed.coverage.deferred.length, 3);
+ assert.deepEqual(source.resolvedDeferred, [
+ { id: "1", reason: "The original review finished." },
+ ]);
+}
+
async function testDiscoveryReceipts(root: string) {
const result = draft([], {
coverage: {
@@ -138,10 +181,13 @@ async function testDiscoveryReceipts(root: string) {
const [retained] = accepted.coverage.surfaces;
assert.equal(retained.id, "api");
assert.deepEqual(retained.receiptRefs, [
- "artifacts/archived-receipts/attempt-01/artifacts/review.md",
+ workerReceiptRef(
+ output,
+ "artifacts/archived-receipts/attempt-01/artifacts/review.md",
+ ),
]);
assert.equal(
- await readFile(path.join(output, retained.receiptRefs[0]), "utf8"),
+ await readFile(receiptPath(output, retained.receiptRefs[0]), "utf8"),
"Original attempt review evidence.\n",
);
assert.equal(
@@ -162,7 +208,7 @@ async function testDiscoveryReceipts(root: string) {
assert.equal(retried.coverage.surfaces.length, 1);
for (const ref of retried.coverage.surfaces[0].receiptRefs) {
assert.equal(
- await readFile(path.join(output, ref), "utf8"),
+ await readFile(receiptPath(output, ref), "utf8"),
"Original attempt review evidence.\n",
);
}
@@ -278,10 +324,13 @@ async function testArchivedReceiptPathValidation(
if (includeValidCheckpoint) {
const refs = accepted.coverage.surfaces[0].receiptRefs;
assert.deepEqual(refs, [
- "artifacts/archived-receipts/attempt-01/artifacts/review.md",
+ workerReceiptRef(
+ output,
+ "artifacts/archived-receipts/attempt-01/artifacts/review.md",
+ ),
]);
assert.equal(
- await readFile(path.join(output, refs[0]), "utf8"),
+ await readFile(receiptPath(output, refs[0]), "utf8"),
"Original review.\n",
);
} else {
@@ -466,7 +515,9 @@ async function testMissingReceiptRepair(
assert.deepEqual(
new Set(
await Promise.all(
- receipts.map((ref: string) => readFile(path.join(output, ref), "utf8")),
+ receipts.map((ref: string) =>
+ readFile(receiptPath(output, ref), "utf8"),
+ ),
),
),
new Set(["Original valid evidence.\n", "Corrected review evidence.\n"]),
@@ -504,7 +555,7 @@ async function testMissingReceiptRepair(
new Set(
await Promise.all(
retried.coverage.surfaces[0].receiptRefs.map((ref: string) =>
- readFile(path.join(output, ref), "utf8"),
+ readFile(receiptPath(output, ref), "utf8"),
),
),
),
@@ -894,10 +945,13 @@ async function testReducerValidation(root: string) {
const validatedSnapshot = await validateSnapshot();
assert.equal(validatedSnapshot.newFindings, 2);
const admitted = await readJson(resultPath);
+ const { sourceCoverage, ...persistedResult } = validatedSnapshot.result;
+ assert.ok(sourceCoverage);
+ assert.equal(sourceCoverage.completeness, undefined);
assert.deepEqual(
- validatedSnapshot.result,
+ persistedResult,
admitted,
- "validation returns the same reconciled result that was accepted on disk",
+ "accepted findings retain the legacy persisted shape while coverage stays host-owned",
);
assert.equal(Object.hasOwn(admitted, "coverage"), false);
assert.deepEqual(admitted.findings[1].provenance.sourceFindingIds, [
@@ -1425,7 +1479,7 @@ async function testSharedMissingReceiptRepairHistory(root: string) {
assert.equal(row.disposition, "no_issue_found");
assert.equal(row.receiptRefs.length, 1);
const content = await readFile(
- path.join(output, row.receiptRefs[0]),
+ receiptPath(output, row.receiptRefs[0]),
"utf8",
);
assert.equal(
@@ -1438,7 +1492,7 @@ async function testSharedMissingReceiptRepairHistory(root: string) {
const retainedWriter = accepted.coverage.surfaces.find(
(row: JsonObject) => row.id === "writer",
)!;
- const retainedPath = path.join(output, retainedWriter.receiptRefs[0]);
+ const retainedPath = receiptPath(output, retainedWriter.receiptRefs[0]);
await rm(retainedPath);
await recordCodexSecurityWorkerScanDraft(context, accepted);
await validateDiscoveryArtifacts(worker.artifacts, worker.resultPath, scanId);
@@ -1506,7 +1560,9 @@ async function testCurrentAnonymousReceiptSelection(
"Replacement review.\n",
);
const originalReceipts = await Promise.all(
- [selectedRef, historicalRef].map((ref) => readFile(path.join(output, ref))),
+ [selectedRef, historicalRef].map((ref) =>
+ readFile(receiptPath(output, ref)),
+ ),
);
const context = {
root: output,
@@ -1552,12 +1608,16 @@ async function testCurrentAnonymousReceiptSelection(
const selected = accepted.coverage.surfaces.find(
(row: JsonObject) => row.label === "Current anonymous review",
);
- assert.deepEqual(selected.receiptRefs, [selectedRef]);
+ assert.deepEqual(selected.receiptRefs, [
+ workerReceiptRef(output, selectedRef),
+ ]);
assert.equal(selected.disposition, "no_issue_found");
const writer = accepted.coverage.surfaces.find(
(row: JsonObject) => row.id === "writer",
);
- assert.deepEqual(writer.receiptRefs, ["artifacts/replacement.md"]);
+ assert.deepEqual(writer.receiptRefs, [
+ workerReceiptRef(output, "artifacts/replacement.md"),
+ ]);
}
assert.deepEqual(
await Promise.all(
@@ -1570,7 +1630,7 @@ async function testCurrentAnonymousReceiptSelection(
assert.deepEqual(
await Promise.all(
[selectedRef, historicalRef].map((ref) =>
- readFile(path.join(output, ref)),
+ readFile(receiptPath(output, ref)),
),
),
originalReceipts,
@@ -1682,17 +1742,23 @@ async function testCurrentReceiptSelectionBeforeArchivedHistory(
(row: JsonObject) => row.id === keptId,
);
assert.deepEqual(kept.receiptRefs, [
- `artifacts/archived-receipts/attempt-01/${refs[keptId]}`,
+ workerReceiptRef(
+ output,
+ `artifacts/archived-receipts/attempt-01/${refs[keptId]}`,
+ ),
]);
assert.equal(
- await readFile(path.join(output, kept.receiptRefs[0]), "utf8"),
+ await readFile(receiptPath(output, kept.receiptRefs[0]), "utf8"),
await readFile(path.join(archive, refs[keptId]), "utf8"),
);
const repaired = result.coverage.surfaces.find(
(row: JsonObject) => row.id === repairId,
);
if (candidate) assert.equal(repaired, undefined);
- else assert.deepEqual(repaired.receiptRefs, ["artifacts/replacement.md"]);
+ else
+ assert.deepEqual(repaired.receiptRefs, [
+ workerReceiptRef(output, "artifacts/replacement.md"),
+ ]);
assert.equal(
await readFile(path.join(archive, "result.json"), "utf8"),
oldResult,
@@ -1715,3 +1781,18 @@ function reduction<Finding extends Record<string, unknown>>(
) {
return { scanId, findings, ...extra };
}
+
+function workerReceiptRef(
+ output: string,
+ ref: string,
+ directory = "output",
+): string {
+ return `artifacts/deep_discovery/workers/${path.basename(path.dirname(output))}/${directory}/${ref}`;
+}
+
+function receiptPath(output: string, ref: string): string {
+ const root = ref.startsWith("artifacts/deep_discovery/workers/")
+ ? path.resolve(output, "../../../../..")
+ : output;
+ return path.join(root, ref);
+}
diff --git a/plugins/codex-security/mcp-app/tests/test_deep_scan_coordinator.ts b/plugins/codex-security/mcp-app/tests/test_deep_scan_coordinator.ts
index c5599f781..b3807ef05 100644
--- a/plugins/codex-security/mcp-app/tests/test_deep_scan_coordinator.ts
+++ b/plugins/codex-security/mcp-app/tests/test_deep_scan_coordinator.ts
@@ -1,12 +1,17 @@
+import { testSharedReceiptSurvivesWorkerRetry } from "./deep_scan_receipt_cases.ts";
+import { testSuccessfulDeepCoveragePreservesWorkerReviewStatus } from "./deep_scan_coverage_cases.ts";
import { readJson, writeJson } from "./support/json.ts";
import type { CoordinatorOptions } from "../src/deep-scan/coordinator.js";
import type { ScanDraftInput } from "../src/artifact-scan-draft.js";
+import type { JsonObject } from "../src/types.js";
import type {
DeepScanConfig,
DeepScanRunState,
CodexWorkerDiagnostic,
PersistedDeepScanWorker,
DeepScanLogEvent,
+ CodexWorkerRequest,
+ CodexWorkerResult,
} from "../src/deep-scan/types.js";
import { mock } from "node:test";
import assert from "node:assert/strict";
@@ -48,6 +53,7 @@ import {
createCoordinator,
runCoordinator,
recordingClock,
+ recordCodexSecurityWorkerScanDraft,
type TestWorker,
type StoreInput,
} from "./deep_scan_coordinator_fixture.ts";
@@ -4252,6 +4258,7 @@ try {
await testCompletionOrdering();
await testSaturationPreservesFindingAlreadyBuffered();
await testSaturationOmitsWorkerAcceptedDuringCancellation();
+ await testSuccessfulDeepCoveragePreservesWorkerReviewStatus();
await testSuccessfulDeepCoverageRetainsIndependentWorkerReviewStatus();
await testSaturationIgnoresDiscoveryCancellationWriteFailure();
await testSaturationIgnoresDiscoveryCancellationWriteFailure(true);
@@ -4288,6 +4295,7 @@ try {
await testMissingDiscoveryResultResumesExistingThread();
await testMissingDiscoveryResultResumesExistingThread(true);
await testInvalidArtifactsRetry();
+ await testSharedReceiptSurvivesWorkerRetry();
await testInvalidReducerResultRetriesFromSnapshot();
await testInvalidReducerResultRetriesFromSnapshot(true);
await testMissingReducerResultResumesExistingThread();
diff --git a/plugins/codex-security/mcp-app/tests/test_deep_scan_executor.ts b/plugins/codex-security/mcp-app/tests/test_deep_scan_executor.ts
index 9feb2ff04..f331fd504 100644
--- a/plugins/codex-security/mcp-app/tests/test_deep_scan_executor.ts
+++ b/plugins/codex-security/mcp-app/tests/test_deep_scan_executor.ts
@@ -1,3 +1,4 @@
+import { testArtifactStartupTimeoutClassification } from "./deep_scan_startup_cases.ts";
import { windowsAcl } from "./support/windows-output-fixture.ts";
import {
createFakeCodexFixture,
@@ -159,6 +160,7 @@ try {
);
await check(t, testExternalProviderCredentialsReachWorker);
await check(t, testArtifactServerUsesExtendedStartupTimeout);
+ await check(t, testReducerAttemptContext);
await check(t, testZeroSubagentsPreservesHostRestrictions);
await check(t, testSdkResumesExistingThread);
}
@@ -227,7 +229,11 @@ try {
await check(t, testPolicyFailuresRemainWorkerErrors);
await check(t, testMalformedCommandEventsRemainRetryable);
await check(t, testRateLimitPolicyFailureRemainsRetryable);
- await check(t, testArtifactStartupTimeoutClassification);
+ await check(
+ t,
+ () => testArtifactStartupTimeoutClassification(runFixtureWorker),
+ testArtifactStartupTimeoutClassification.name,
+ );
}
});
} finally {
@@ -2951,6 +2957,63 @@ async function testArtifactServerUsesExtendedStartupTimeout() {
});
}
+async function testReducerAttemptContext() {
+ const fixture = await fakeCodexFixture();
+ const previousPath = process.env.CODEX_CLI_PATH;
+ process.env.CODEX_CLI_PATH = fixture.executablePath;
+ try {
+ const promptPath = path.join(fixture.root, "prompt.md");
+ const workingDirectory = path.join(fixture.root, "artifacts");
+ await mkdir(workingDirectory);
+ await writeFile(promptPath, "fixture reducer prompt\n");
+ const deepReducer = {
+ scanRoot: path.join(fixture.root, "scans"),
+ claimedWorkers: [
+ {
+ id: "worker-1",
+ resultPath: path.join(fixture.root, "worker", "result.json"),
+ attempt: 2,
+ },
+ ],
+ };
+ for (const resume of [false, true]) {
+ await new CodexSdkWorkerExecutor({
+ parentSandbox: trustedParentSandbox,
+ artifactContext: {
+ pluginRoot: fixture.root,
+ scanRoot: deepReducer.scanRoot,
+ repoRoot: fixture.root,
+ scanId: "fixture-scan-id",
+ },
+ }).run({
+ kind: "dedup",
+ promptPath,
+ workingDirectory,
+ subagents: 0,
+ signal: new AbortController().signal,
+ ...(resume
+ ? {
+ resumeThreadId: "fixture-existing-thread",
+ continuationPrompt: "continue the reducer\n",
+ }
+ : {}),
+ artifactContext: {
+ root: workingDirectory,
+ layout: "reducer",
+ deepReducer,
+ },
+ });
+ const invocation = JSON.parse(await readFile(fixture.markerPath, "utf8"));
+ assertConfigOverrides(invocation.argv, {
+ "mcp_servers.cs_artifacts.env.CODEX_SECURITY_REDUCER_CONTEXT_JSON":
+ JSON.stringify(deepReducer),
+ });
+ }
+ } finally {
+ restoreEnv("CODEX_CLI_PATH", previousPath);
+ }
+}
+
async function testSdkResumesExistingThread() {
return withWorkerFixture(async (fixture, promptPath, workingDirectory) => {
await writeFile(promptPath, "original worker prompt\n");
@@ -3623,35 +3686,6 @@ async function testMalformedCommandEventsRemainRetryable() {
}
}
-async function testArtifactStartupTimeoutClassification() {
- for (const prompt of [
- "ARTIFACT_MCP_STARTUP_TIMEOUT",
- "LEGACY_ARTIFACT_MCP_STARTUP_TIMEOUT",
- "ARTIFACT_MCP_STARTUP_TIMEOUT_REQUEST_TIMED_OUT",
- "ARTIFACT_MCP_STARTUP_TIMEOUT_SYNC_AUTH_WARNING",
- "LEGACY_ARTIFACT_MCP_STARTUP_TIMEOUT_SYNC_AUTH_WARNING",
- "ARTIFACT_MCP_STARTUP_TIMEOUT_BOTH_AUTH_WARNINGS",
- "LEGACY_ARTIFACT_MCP_STARTUP_TIMEOUT_BOTH_AUTH_WARNINGS",
- "ARTIFACT_MCP_STARTUP_TIMEOUT_REQUEST_TIMED_OUT_SYNC_AUTH_WARNING",
- "ARTIFACT_MCP_STARTUP_TIMEOUT_REQUEST_TIMED_OUT_BOTH_AUTH_WARNINGS",
- "ARTIFACT_MCP_STARTUP_TIMEOUT_WITH_MISSING_API_KEY",
- "ARTIFACT_MCP_STARTUP_TIMEOUT_WITH_POLICY_REFUSAL",
- "ARTIFACT_MCP_STARTUP_TIMEOUT_REQUEST_TIMED_OUT_SYNC_AUTH_WARNING_WITH_MISSING_API_KEY",
- "ARTIFACT_MCP_STARTUP_TIMEOUT_REQUEST_TIMED_OUT_BOTH_AUTH_WARNINGS_WITH_POLICY_REFUSAL",
- "OTHER_MCP_STARTUP_TIMEOUT",
- "OTHER_MCP_STARTUP_TIMEOUT_REQUEST_TIMED_OUT_SYNC_AUTH_WARNING",
- "CATALOG_AUTH_ONLY",
- "SYNC_AUTH_ONLY",
- ]) {
- await assert.rejects(
- runFixtureWorker(prompt),
- (error: NodeJS.ErrnoException & { cause?: NodeJS.ErrnoException }) =>
- error?.name === "Error",
- `${prompt} should remain retryable`,
- );
- }
-}
-
async function testMissingParentSandboxFailsBeforeWorkerLaunch() {
const fixture = await fakeCodexFixture();
const previousPath = process.env.CODEX_CLI_PATH;
diff --git a/plugins/codex-security/mcp-app/tests/test_deep_scan_publication_integration.ts b/plugins/codex-security/mcp-app/tests/test_deep_scan_publication_integration.ts
index c208b9957..b826f2f2f 100644
--- a/plugins/codex-security/mcp-app/tests/test_deep_scan_publication_integration.ts
+++ b/plugins/codex-security/mcp-app/tests/test_deep_scan_publication_integration.ts
@@ -1038,6 +1038,17 @@ for (const workflow of ["deep-scan-mcp/v1", "deep-security-scan/v1"]) {
if (receiptIoFailure)
f.environment.TEST_WORKBENCH_RECEIPT_IO_FAILURE = "1";
const before = await f.workers();
+ const expectedReviews = before.workers
+ .filter(
+ (worker) =>
+ worker.kind === "discovery" && worker.status === "succeeded",
+ )
+ .map((worker) => ({ workerId: worker.id, attempt: worker.attempt }));
+ assert.equal(expectedReviews.length, 4);
+ const expectedDraft = {
+ ...f.draft,
+ coverage: { ...f.draft.coverage, reviews: expectedReviews },
+ };
const sources = await Promise.all(
f.results.map((file) => readFile(file)),
);
@@ -1071,7 +1082,7 @@ for (const workflow of ["deep-scan-mcp/v1", "deep-security-scan/v1"]) {
clock: { now: () => Date.parse(f.instant), sleep: async () => {} },
onComplete: async (draft, signal, publication) => {
publications++;
- assert.deepEqual(draft, f.draft);
+ assert.deepEqual(draft, expectedDraft);
assert.deepEqual(publication, {
coordinatorGeneration: 2,
resultPath: f.results[1],
diff --git a/plugins/codex-security/mcp-app/tests/test_deep_scan_store_integration.ts b/plugins/codex-security/mcp-app/tests/test_deep_scan_store_integration.ts
index 301bac885..f320b128d 100644
--- a/plugins/codex-security/mcp-app/tests/test_deep_scan_store_integration.ts
+++ b/plugins/codex-security/mcp-app/tests/test_deep_scan_store_integration.ts
@@ -1126,7 +1126,7 @@ async function testPartialWorkerCoverageSurvivesParentCompletion() {
assert.equal(terminal?.status, "succeeded", terminal?.error);
await runWorkbench(["complete-scan", "--scan-id", run.scanId]);
const coverage = await readJson(run.scanDir, "coverage.json");
- for (const field of ["surfaces", "explicitExclusions", "deferred"]) {
+ for (const field of ["surfaces", "deferred"]) {
assert.equal(coverage[field].length, 4, field);
assert.equal(
new Set(coverage[field].map((row: { id: string }) => row.id)).size,
@@ -1134,12 +1134,25 @@ async function testPartialWorkerCoverageSurvivesParentCompletion() {
field,
);
}
+ assert.equal(coverage.explicitExclusions.length, 4);
+ assert.equal(
+ new Set(
+ coverage.explicitExclusions.map(
+ (row: { pattern: string; provenance: { workerId: string } }) =>
+ `${row.provenance.workerId}/${row.pattern}`,
+ ),
+ ).size,
+ 4,
+ "independent worker exclusions retain their provenance",
+ );
assert.equal(coverage.openQuestions.length, 2);
for (const row of coverage.surfaces) {
if (row.label !== "Request entry") continue;
+ assert.equal(row.provenance.sourceId, "1");
+ assert.equal(row.provenance.attempt, 1);
assert.equal(
row.id,
- `${row.workerId}/1`,
+ `${row.provenance.workerId}-attempt-1-surface-1`,
"explicit source IDs retain their links",
);
assert.equal(
diff --git a/plugins/codex-security/mcp-app/tests/test_parent_candidate_followup.ts b/plugins/codex-security/mcp-app/tests/test_parent_candidate_followup.ts
new file mode 100644
index 000000000..1291da8df
--- /dev/null
+++ b/plugins/codex-security/mcp-app/tests/test_parent_candidate_followup.ts
@@ -0,0 +1,167 @@
+import assert from "node:assert/strict";
+import { execFile } from "node:child_process";
+import { mkdir, readFile, rm, writeFile } from "node:fs/promises";
+import path from "node:path";
+import test from "node:test";
+import { promisify } from "node:util";
+import { importSource } from "./import-module.ts";
+import { temporaryDirectory } from "./support/temporary-directories.ts";
+
+const exec = promisify(execFile);
+const pluginRoot = path.resolve(import.meta.dirname, "../..");
+const { recordCodexSecurityScanDraftViaWorkbench } = await importSource(
+ "../src/artifact-scan-draft.ts",
+ { absWorkingDir: import.meta.dirname },
+);
+const { createScanArtifactContext } = await importSource(
+ "../src/artifact-context.ts",
+ { absWorkingDir: import.meta.dirname },
+);
+
+for (const newerTask of [false, true]) {
+ for (const stoppedRetry of [false, true]) {
+ test(`parent candidate follow-up: newer task=${newerTask}, stopped retry=${stoppedRetry}`, async (t) => {
+ const root = await temporaryDirectory("parent-candidate-followup-", true);
+ t.after(() => rm(root, { recursive: true, force: true }));
+ const target = path.join(root, "target");
+ const scanDir = path.join(root, "scan");
+ await mkdir(path.join(target, "src"), { recursive: true });
+ await mkdir(scanDir, { mode: 0o700 });
+ await writeFile(
+ path.join(target, "src/example.py"),
+ "# Synthetic source\n",
+ );
+ const environment = {
+ ...process.env,
+ CODEX_HOME: path.join(root, "home"),
+ CODEX_SECURITY_STATE_DIR: path.join(root, "state"),
+ };
+ const script = path.join(pluginRoot, "scripts/workbench_db.py");
+ const runWorkbench = async (args: string[]) => {
+ const { stdout } = await exec(
+ process.env.PYTHON || "python3",
+ [script, ...args],
+ { env: environment },
+ );
+ return JSON.parse(stdout);
+ };
+ const registered = await runWorkbench([
+ "register-cli-scan",
+ "--repository",
+ target,
+ "--scan-dir",
+ scanDir,
+ "--recipe-json",
+ JSON.stringify({
+ config: {},
+ mode: "standard",
+ repository: target,
+ target: { kind: "repository", paths: [] },
+ }),
+ ]);
+ const scanId = registered.scanId;
+ const context = await createScanArtifactContext(scanId, runWorkbench, {
+ requireRunning: true,
+ });
+ const finding = {
+ ruleId: "fixture.review",
+ identity: { anchor: "synthetic-review" },
+ title: "Synthetic review finding",
+ summary: "Retained evidence remains available.",
+ severity: { level: "low" },
+ confidence: {
+ level: "high",
+ rationale: "Synthetic persistence fixture.",
+ },
+ taxonomy: { category: "other", cwe: [] },
+ locations: [{ path: "src/example.py", startLine: 1 }],
+ remediation: "Complete the review.",
+ provenance: { source: "local_plugin", candidateId: "candidate-c" },
+ };
+ const task = {
+ id: "candidate-followup",
+ candidateId: "candidate-c",
+ reason: "Review the later caller.",
+ paths: ["src/example.py"],
+ };
+ const coverage = {
+ completeness: "partial",
+ surfaces: [],
+ explicitExclusions: [],
+ deferred: [],
+ };
+ await recordCodexSecurityScanDraftViaWorkbench(
+ context,
+ {
+ scanId,
+ complete: false,
+ findings: newerTask ? [finding] : [],
+ coverage: { ...coverage, deferred: newerTask ? [] : [task] },
+ },
+ runWorkbench,
+ );
+ const written = await recordCodexSecurityScanDraftViaWorkbench(
+ context,
+ {
+ scanId,
+ complete: true,
+ findings: newerTask ? [] : [finding],
+ coverage: {
+ ...coverage,
+ completeness: newerTask ? "partial" : "complete",
+ deferred: newerTask ? [task] : [],
+ },
+ },
+ runWorkbench,
+ );
+ assert.equal(written.findingCount, 1);
+ assert.deepEqual(written.coverage.deferred, newerTask ? [task] : []);
+ if (stoppedRetry) {
+ const fault = path.join(root, "fault.py");
+ await writeFile(
+ fault,
+ `import sys\nsys.path.insert(0, ${JSON.stringify(path.join(pluginRoot, "scripts"))})\nimport workbench_db, workbench_saved_results\ndef fail(*args, **kwargs):\n raise OSError("Synthetic publication interruption.")\nworkbench_saved_results._write_prepared_scan_finalization = fail\nworkbench_db.main()\n`,
+ );
+ const { stdout } = await exec(
+ process.env.PYTHON || "python3",
+ [
+ fault,
+ "fail-scan",
+ "--scan-id",
+ scanId,
+ "--message",
+ "Synthetic stop.",
+ ],
+ { env: environment },
+ );
+ assert.equal(JSON.parse(stdout).scan.resultsRecoveryNeeded, true);
+ await runWorkbench(["recover-scan-results", "--scan-id", scanId]);
+ } else {
+ await runWorkbench(["complete-scan", "--scan-id", scanId]);
+ }
+ const saved = (await runWorkbench(["get-scan", "--scan-id", scanId]))
+ .scan;
+ assert.equal(saved.resultsRecoveryNeeded, false);
+ assert.equal(saved.findingCount, 1);
+ const published = JSON.parse(
+ await readFile(path.join(scanDir, "coverage.json"), "utf8"),
+ );
+ assert.deepEqual(
+ published.deferred.filter(
+ (row: { id: string }) => row.id !== "scan-stopped",
+ ),
+ newerTask ? [task] : [],
+ );
+ const bytes = await readFile(path.join(scanDir, "coverage.json"), "utf8");
+ await runWorkbench([
+ stoppedRetry ? "recover-scan-results" : "get-scan",
+ "--scan-id",
+ scanId,
+ ]);
+ assert.equal(
+ await readFile(path.join(scanDir, "coverage.json"), "utf8"),
+ bytes,
+ );
+ });
+ }
+}
diff --git a/plugins/codex-security/mcp-app/tests/test_receipt_ownership.ts b/plugins/codex-security/mcp-app/tests/test_receipt_ownership.ts
new file mode 100644
index 000000000..c5460540c
--- /dev/null
+++ b/plugins/codex-security/mcp-app/tests/test_receipt_ownership.ts
@@ -0,0 +1,1051 @@
+import assert from "node:assert/strict";
+import { execFileSync } from "node:child_process";
+import {
+ mkdir,
+ readFile,
+ readdir,
+ rename,
+ rm,
+ utimes,
+ writeFile,
+} from "node:fs/promises";
+import path from "node:path";
+import test from "node:test";
+import { importSource } from "./import-module.ts";
+import {
+ claimDedupInputs,
+ FakeExecutor,
+ FakeStore,
+ fixtureRun,
+ runCoordinator,
+} from "./deep_scan_coordinator_fixture.ts";
+import { finding, scanId, workerDraft } from "./scan-draft-fixture.ts";
+import { temporaryDirectory } from "./support/temporary-directories.ts";
+
+const { readDeepReductionSources } = await importSource(
+ path.join(import.meta.dirname, "../src/artifact-deep-reducer.ts"),
+);
+const { recordCodexSecurityWorkerScanDraft, saveScanDraftCheckpoint } =
+ await importSource(
+ path.join(import.meta.dirname, "../src/artifact-scan-draft.ts"),
+ );
+
+const { archiveDirectory } = await importSource(
+ path.join(import.meta.dirname, "../src/deep-scan/artifacts.ts"),
+);
+
+async function fixture() {
+ const root = await temporaryDirectory("receipt-ownership-", true);
+ const prefix = "artifacts/deep_discovery/workers/discovery-0001/output";
+ const workerRoot = path.join(root, prefix);
+ const reducerRoot = path.join(
+ root,
+ "artifacts/deep_discovery/dedup/run/output",
+ );
+ await mkdir(workerRoot, { recursive: true });
+ await mkdir(reducerRoot, { recursive: true });
+ return {
+ root,
+ prefix,
+ workerRoot,
+ worker: { root: workerRoot, repoRoot: root, scanId, layout: "worker" },
+ reducer: {
+ root: reducerRoot,
+ repoRoot: root,
+ scanId,
+ layout: "reducer",
+ deepReducer: {
+ scanRoot: root,
+ claimedWorkers: [
+ { id: "worker-1", resultPath: path.join(workerRoot, "result.json") },
+ ],
+ },
+ },
+ };
+}
+
+function draft(receiptRefs: string[], extra = {}) {
+ return workerDraft([], {
+ coverage: {
+ ...workerDraft([]).coverage,
+ surfaces: [
+ {
+ id: "reviewed-route",
+ label: "Reviewed route",
+ disposition: "no_issue_found",
+ receiptRefs,
+ ...extra,
+ },
+ ],
+ },
+ });
+}
+
+async function receipt(root: string, ref: string, contents: string) {
+ await mkdir(path.dirname(path.join(root, ref)), { recursive: true });
+ await writeFile(path.join(root, ref), contents);
+}
+
+function recoverCoverage(
+ f: Awaited<ReturnType<typeof fixture>>,
+ complete = true,
+ attempt = 1,
+ includePaths = ["."],
+ publishDuringCapture = false,
+ explicitRecovery = false,
+) {
+ return JSON.parse(
+ execFileSync(
+ process.env.PYTHON || "python3",
+ [
+ "-B",
+ "-c",
+ `
+import json, sys
+from types import SimpleNamespace
+from pathlib import Path
+plugin, root, active = map(Path, sys.argv[1:4])
+scan_id, complete = sys.argv[4], sys.argv[5] == "true"
+sys.path[:0] = [str(plugin / "scripts"), str(plugin / "tests")]
+import workbench_saved_results as saved
+import workbench.retained_coverage as retained
+from workbench_saved_results import merge_saved_results
+from workbench_test_support import saved_binding, saved_discovery_worker
+from finalize_scan_contract import _recover_unsealed_coverage
+worker = {**saved_discovery_worker(active, "worker-1", int(sys.argv[6])), "status": "succeeded" if complete else "running", "result_manifest_path": str(active / "result.json") if complete else None}
+binding = saved_binding("deep_repository")
+binding["target"].update(targetId="synthetic", displayName="Synthetic repository")
+binding["scope"]["includePaths"] = json.loads(sys.argv[7])
+args = (root, scan_id, binding, [worker], [])
+original_capture = saved._capture_saved_source
+original_retained_capture = retained._capture_saved_source
+published = []
+def capture(*args, **kwargs):
+ captured = original_capture(*args, **kwargs)
+ if sys.argv[8] == "true" and args[1].endswith("output/checkpoint-head.json") and not published:
+ staged = root / "publication"
+ for checkpoint in (staged / "checkpoints").iterdir():
+ destination = active / "checkpoints" / checkpoint.name
+ checkpoint.replace(destination)
+ published.append(destination.relative_to(root).as_posix())
+ for name in ("checkpoint-head.json", "result.json"):
+ (staged / name).replace(active / name)
+ return captured
+saved._capture_saved_source = capture
+retained._capture_saved_source = capture
+frozen = None
+if sys.argv[9] == "true":
+ db = SimpleNamespace(require_canonical_scan_directory=lambda path: path, artifact_path=lambda *args, **kwargs: None, ARTIFACTS={"manifest": "scan-manifest.json"})
+ connection = SimpleNamespace(execute=lambda *args: SimpleNamespace(fetchall=lambda: [worker]))
+ scan = {"scan_dir": str(root), "id": scan_id, "retained_source_digests_json": None, "seal_manifest_digest": None}
+ frozen, _ = saved._recovery_source_digests(db, connection, scan)
+merged = merge_saved_results(*args, stopped=True, reason="interrupted", frozen_source_digests=frozen)
+saved._capture_saved_source = original_capture
+retained._capture_saved_source = original_retained_capture
+replay = merge_saved_results(*args, stopped=True, reason="interrupted", frozen_source_digests=merged[0]["scan"]["preservedSources"])
+assert merged[2] == replay[2]
+rejects_changed_checkpoint = None
+if published:
+ checkpoint = root / published[-1]
+ original = checkpoint.read_bytes()
+ checkpoint.write_text("{}")
+ try:
+ merge_saved_results(*args, stopped=True, reason="interrupted", frozen_source_digests=merged[0]["scan"]["preservedSources"])
+ except saved.ContractError:
+ rejects_changed_checkpoint = True
+ else:
+ rejects_changed_checkpoint = False
+ finally:
+ checkpoint.write_bytes(original)
+refs = merged[2]["surfaces"][0]["receiptRefs"]
+warnings = []
+coverage = merged[2]
+_recover_unsealed_coverage(coverage, plugin / "schemas", root, warnings, [])
+print(json.dumps({"refs": refs, "surfaces": coverage["surfaces"], "warnings": warnings, "published": published, "captured": merged[0]["scan"]["preservedSources"], "rejectsChangedCheckpoint": rejects_changed_checkpoint}))
+`,
+ path.resolve(import.meta.dirname, "../.."),
+ f.root,
+ f.workerRoot,
+ scanId,
+ String(complete),
+ String(attempt),
+ JSON.stringify(includePaths),
+ String(publishDuringCapture),
+ String(explicitRecovery),
+ ],
+ {
+ encoding: "utf8",
+ ...(explicitRecovery
+ ? { env: { ...process.env, PYTHONHASHSEED: "0" } }
+ : {}),
+ },
+ ),
+ );
+}
+
+for (const sharedExists of [false, true]) {
+ test(`direct worker provenance cannot change local receipt ownership (shared exists: ${sharedExists})`, async () => {
+ const f = await fixture();
+ try {
+ const ref = "artifacts/evidence.txt";
+ await receipt(f.workerRoot, ref, "Local evidence\n");
+ if (sharedExists)
+ await receipt(f.root, ref, "Unrelated shared evidence\n");
+ await writeFile(
+ path.join(f.workerRoot, "result.json"),
+ JSON.stringify(
+ draft([ref], { provenance: { scanReceiptRefs: [ref] } }),
+ ),
+ );
+ const sources = await readDeepReductionSources(f.reducer);
+ const [projected] = sources.discoveries[0].coverage.surfaces;
+ assert.deepEqual(projected.receiptRefs, [`${f.prefix}/${ref}`]);
+ assert.equal(
+ await readFile(path.join(f.root, projected.receiptRefs[0]), "utf8"),
+ "Local evidence\n",
+ );
+ } finally {
+ await rm(f.root, { recursive: true, force: true });
+ }
+ });
+}
+
+test("direct worker receipts fall back to an existing shared file without a local receipt", async () => {
+ const f = await fixture();
+ try {
+ const ref = "artifacts/evidence.txt";
+ await receipt(f.root, ref, "Shared evidence\n");
+ await writeFile(
+ path.join(f.workerRoot, "result.json"),
+ JSON.stringify(draft([ref])),
+ );
+ const sources = await readDeepReductionSources(f.reducer);
+ assert.deepEqual(sources.discoveries[0].coverage.surfaces[0].receiptRefs, [
+ ref,
+ ]);
+ } finally {
+ await rm(f.root, { recursive: true, force: true });
+ }
+});
+
+for (const update of ["notes", "receipt order"] as const) {
+ test(`retained shared ownership survives changed ${update} and a local collision`, async () => {
+ const f = await fixture();
+ try {
+ const refs = ["artifacts/first.txt", "artifacts/second.txt"];
+ for (const ref of refs) await receipt(f.root, ref, "Shared evidence\n");
+ await recordCodexSecurityWorkerScanDraft(f.worker, draft(refs));
+ for (const ref of refs)
+ await receipt(f.workerRoot, ref, "Unrelated local evidence\n");
+ await recordCodexSecurityWorkerScanDraft(
+ f.worker,
+ draft(
+ update === "receipt order" ? [...refs].reverse() : refs,
+ update === "notes" ? { notes: "Updated review notes." } : {},
+ ),
+ );
+ const sources = await readDeepReductionSources(f.reducer);
+ const [projected] = sources.discoveries[0].coverage.surfaces;
+ assert.deepEqual([...projected.receiptRefs].sort(), [...refs].sort());
+ for (const ref of projected.receiptRefs)
+ assert.equal(
+ await readFile(path.join(f.root, ref), "utf8"),
+ "Shared evidence\n",
+ );
+ } finally {
+ await rm(f.root, { recursive: true, force: true });
+ }
+ });
+}
+
+for (const changed of ["result provenance", "checkpoint contents"] as const) {
+ test(`saved ownership requires the exact checkpoint payload after changed ${changed}`, async () => {
+ const f = await fixture();
+ try {
+ const ref = "artifacts/evidence.txt";
+ await receipt(f.root, ref, "Shared evidence\n");
+ await recordCodexSecurityWorkerScanDraft(f.worker, draft([ref]));
+ await receipt(f.workerRoot, ref, "Local evidence\n");
+ const resultPath = path.join(f.workerRoot, "result.json");
+ if (changed === "result provenance") {
+ const saved = JSON.parse(await readFile(resultPath, "utf8"));
+ saved.coverage.surfaces[0].provenance.extra =
+ "Directly authored extension";
+ await writeFile(resultPath, JSON.stringify(saved));
+ } else {
+ const checkpointRoot = path.join(f.workerRoot, "checkpoints");
+ for (const name of await readdir(checkpointRoot)) {
+ const checkpoint = path.join(checkpointRoot, name);
+ const saved = JSON.parse(await readFile(checkpoint, "utf8"));
+ saved.coverage.surfaces[0].provenance.extra = "Changed checkpoint";
+ await writeFile(checkpoint, JSON.stringify(saved));
+ }
+ }
+ const sources = await readDeepReductionSources(f.reducer);
+ assert.deepEqual(
+ sources.discoveries[0].coverage.surfaces[0].receiptRefs,
+ [`${f.prefix}/${ref}`],
+ );
+ } finally {
+ await rm(f.root, { recursive: true, force: true });
+ }
+ });
+}
+
+for (const fileScope of [false, true]) {
+ for (const inScope of [false, true]) {
+ for (const complete of [false, true]) {
+ test(`scoped worker ${complete ? "publication" : "checkpoint"} retains shared receipts from a legacy draft (finding in scope: ${inScope}, exact file: ${fileScope})`, async () => {
+ const f = await fixture();
+ try {
+ const ref = "artifacts/evidence.txt";
+ await receipt(f.root, ref, "Shared evidence\n");
+ // Older workers saved progress without the requested-path binding.
+ await recordCodexSecurityWorkerScanDraft(f.worker, {
+ ...draft([ref]),
+ findings: [
+ finding("inherited", inScope ? "src/route.ts" : "other/route.ts"),
+ ...(fileScope
+ ? [finding("file-descendant", "src/route.ts/generated")]
+ : []),
+ ],
+ });
+ await receipt(f.workerRoot, ref, "Unrelated local evidence\n");
+ await recordCodexSecurityWorkerScanDraft(
+ {
+ ...f.worker,
+ includePaths: [fileScope ? "src/route.ts" : "src"],
+ filePaths: fileScope ? ["src/route.ts"] : [],
+ },
+ { ...workerDraft([]), complete },
+ );
+ const published = JSON.parse(
+ await readFile(path.join(f.workerRoot, "result.json"), "utf8"),
+ );
+ assert.equal(published.findings.length, inScope ? 1 : 0);
+ const sources = complete
+ ? await readDeepReductionSources(f.reducer)
+ : undefined;
+ const recovered = recoverCoverage(f, complete);
+ assert.deepEqual(
+ {
+ reduction:
+ sources?.discoveries[0].coverage.surfaces[0].receiptRefs,
+ recovery: recovered.refs,
+ },
+ { reduction: complete ? [ref] : undefined, recovery: [ref] },
+ );
+ assert.equal(recovered.surfaces[0].disposition, "no_issue_found");
+ assert.deepEqual(recovered.warnings, []);
+ assert.equal(
+ await readFile(path.join(f.root, recovered.refs[0]), "utf8"),
+ "Shared evidence\n",
+ );
+ const head = JSON.parse(
+ await readFile(
+ path.join(f.workerRoot, "checkpoint-head.json"),
+ "utf8",
+ ),
+ );
+ assert.deepEqual(
+ JSON.parse(
+ await readFile(
+ path.join(f.workerRoot, "checkpoints", head.checkpoint),
+ "utf8",
+ ),
+ ),
+ published,
+ );
+ } finally {
+ await rm(f.root, { recursive: true, force: true });
+ }
+ });
+ }
+ }
+}
+
+for (const scenario of [
+ "scoped",
+ "unscoped",
+ "wrong scope",
+ "changed finding",
+ "changed coverage",
+] as const) {
+ for (const resultIsNewer of [true, false]) {
+ test(`legacy worker receipt ownership: ${scenario} (${resultIsNewer ? "newer result" : "tied observations"})`, async () => {
+ const f = await fixture();
+ try {
+ // Captured from the writer before it checkpointed the final scope filter.
+ const legacy = JSON.parse(
+ await readFile(
+ path.join(
+ import.meta.dirname,
+ "fixtures/legacy-scoped-worker-receipts.json",
+ ),
+ "utf8",
+ ),
+ );
+ const ref = "artifacts/evidence.txt";
+ await receipt(f.root, ref, "Shared evidence\n");
+ await receipt(f.workerRoot, ref, "Unrelated local evidence\n");
+ await saveScanDraftCheckpoint(f.worker, legacy.checkpoint);
+ const result =
+ scenario === "unscoped" ? legacy.checkpoint : legacy.result;
+ if (scenario === "changed finding")
+ result.findings[0].title = "Changed finding";
+ if (scenario === "changed coverage")
+ result.coverage.surfaces[0].notes = "Changed review";
+ await writeFile(
+ path.join(f.workerRoot, "result.json"),
+ JSON.stringify(result),
+ );
+ const head = JSON.parse(
+ await readFile(
+ path.join(f.workerRoot, "checkpoint-head.json"),
+ "utf8",
+ ),
+ );
+ // Filesystems can give sequential writes equal mtimes. Bind the intended
+ // observation order explicitly instead of depending on clock resolution.
+ await Promise.all([
+ utimes(path.join(f.workerRoot, "checkpoints", head.checkpoint), 1, 1),
+ utimes(path.join(f.workerRoot, "checkpoint-head.json"), 1, 1),
+ utimes(
+ path.join(f.workerRoot, "result.json"),
+ resultIsNewer ? 2 : 1,
+ resultIsNewer ? 2 : 1,
+ ),
+ ]);
+ const includePaths =
+ scenario === "unscoped"
+ ? ["."]
+ : scenario === "wrong scope"
+ ? ["other"]
+ : ["src"];
+ const sources = await readDeepReductionSources({
+ ...f.reducer,
+ includePaths,
+ });
+ const recovered = recoverCoverage(f, true, 1, includePaths);
+ const expected =
+ scenario === "scoped" || scenario === "unscoped"
+ ? ref
+ : `${f.prefix}/${ref}`;
+ assert.deepEqual(
+ sources.discoveries[0].coverage.surfaces[0].receiptRefs,
+ [expected],
+ );
+ if (resultIsNewer) {
+ assert.deepEqual(recovered.refs, [expected]);
+ } else {
+ // Recovery keeps the separate trusted checkpoint when observations tie;
+ // direct reducer admission still rejects the unrecorded result above.
+ assert.deepEqual(recovered.refs, [ref]);
+ assert.equal(recovered.surfaces[0].notes, undefined);
+ }
+ } finally {
+ await rm(f.root, { recursive: true, force: true });
+ }
+ });
+ }
+}
+
+for (const explicit of [false, true]) {
+ for (const concurrent of [false, true]) {
+ test(`${explicit ? "explicit recovery" : "cancellation"} captures the checkpoint for a host-published result (concurrent: ${concurrent})`, async () => {
+ const f = await fixture();
+ try {
+ const ref = "artifacts/evidence.txt";
+ await receipt(f.root, ref, "Shared evidence\n");
+ await recordCodexSecurityWorkerScanDraft(f.worker, draft([ref]));
+ const checkpointRoot = path.join(f.workerRoot, "checkpoints");
+ const initialCheckpoints = new Set(await readdir(checkpointRoot));
+ const initial = await Promise.all(
+ ["checkpoint-head.json", "result.json"].map(
+ async (name) =>
+ [name, await readFile(path.join(f.workerRoot, name))] as const,
+ ),
+ );
+ await receipt(f.workerRoot, ref, "Unrelated local evidence\n");
+ await recordCodexSecurityWorkerScanDraft(f.worker, {
+ ...draft([ref]),
+ scope: { summary: "Additional scope context" },
+ });
+ if (concurrent) {
+ const staged = path.join(f.root, "publication");
+ await mkdir(path.join(staged, "checkpoints"), { recursive: true });
+ for (const name of await readdir(checkpointRoot)) {
+ if (!initialCheckpoints.has(name))
+ await rename(
+ path.join(checkpointRoot, name),
+ path.join(staged, "checkpoints", name),
+ );
+ }
+ for (const [name, contents] of initial) {
+ await rename(
+ path.join(f.workerRoot, name),
+ path.join(staged, name),
+ );
+ await writeFile(path.join(f.workerRoot, name), contents);
+ await utimes(path.join(f.workerRoot, name), 1, 1);
+ }
+ }
+ const recovered = recoverCoverage(
+ f,
+ true,
+ 1,
+ ["."],
+ concurrent,
+ explicit,
+ );
+ assert.deepEqual(recovered.refs, [ref]);
+ if (concurrent) {
+ assert.ok(recovered.published.length > 0);
+ assert.ok(
+ recovered.published.every(
+ (name: string) => name in recovered.captured,
+ ),
+ );
+ assert.equal(recovered.rejectsChangedCheckpoint, true);
+ }
+ } finally {
+ await rm(f.root, { recursive: true, force: true });
+ }
+ });
+ }
+}
+
+for (const fileScope of [false, true])
+ for (const localReceipt of [false, true]) {
+ test(`the coordinator binds the requested scope while reducing legacy receipt ownership (exact file: ${fileScope}, local receipt: ${localReceipt})`, async () => {
+ const fixture = await fixtureRun();
+ fixture.run.scope = fileScope ? "src/route.ts" : "src";
+ fixture.run.includePaths = [fixture.run.scope];
+ fixture.run.filePaths = fileScope ? [fixture.run.scope] : [];
+ const store = new FakeStore(fixture.run);
+ const executor = new FakeExecutor();
+ const run = executor.run.bind(executor);
+ const ref = "artifacts/evidence.txt";
+ executor.run = async (request) => {
+ const result = await run(request);
+ if (request.kind === "discovery") {
+ const legacy = JSON.parse(
+ await readFile(
+ path.join(
+ import.meta.dirname,
+ "fixtures/legacy-scoped-worker-receipts.json",
+ ),
+ "utf8",
+ ),
+ );
+ if (fileScope)
+ legacy.checkpoint.findings[1].locations[0].path =
+ "src/route.ts/generated";
+ legacy.checkpoint.scanId = fixture.run.scanId;
+ legacy.result.scanId = fixture.run.scanId;
+ await receipt(fixture.run.scanDir, ref, "Shared evidence\n");
+ if (localReceipt)
+ await receipt(
+ request.workingDirectory,
+ ref,
+ "Unrelated local evidence\n",
+ );
+ await saveScanDraftCheckpoint(
+ {
+ ...request.artifactContext,
+ scanId: fixture.run.scanId,
+ repoRoot: fixture.run.targetPath,
+ },
+ legacy.checkpoint,
+ );
+ await writeFile(
+ path.join(request.workingDirectory, "result.json"),
+ JSON.stringify(legacy.result),
+ );
+ }
+ return result;
+ };
+ try {
+ for (const resume of [false, true]) {
+ if (resume) {
+ store.run = {
+ ...store.run,
+ status: "running",
+ terminalReason: undefined,
+ manifestPath: undefined,
+ persistedWorkers: structuredClone([...store.workers.values()]),
+ persistedDedupInputs: store.dedupClaims.flatMap(claimDedupInputs),
+ };
+ }
+ const activeExecutor = resume ? new FakeExecutor() : executor;
+ const receiptRefs: unknown[] = [];
+ const terminal = await runCoordinator(
+ fixture,
+ store,
+ activeExecutor,
+ {
+ run: store.run,
+ onComplete: async (draft) => {
+ receiptRefs.push(
+ ...(
+ draft.coverage.surfaces as { receiptRefs: string[] }[]
+ ).flatMap((surface) => surface.receiptRefs),
+ );
+ },
+ },
+ );
+ assert.equal(terminal?.status, "succeeded", terminal?.error);
+ assert.equal(activeExecutor.discoveryCalls, resume ? 0 : 1);
+ assert.equal(activeExecutor.dedupCalls, resume ? 0 : 1);
+ assert.deepEqual(receiptRefs, [ref]);
+ }
+ } finally {
+ await rm(path.dirname(fixture.run.scanDir), {
+ recursive: true,
+ force: true,
+ });
+ }
+ });
+ }
+
+test("a retained surface keeps only retained shared references and resolves new receipts locally", async () => {
+ const f = await fixture();
+ try {
+ const retained = "artifacts/retained.txt";
+ const removed = "artifacts/removed.txt";
+ const added = "artifacts/added.txt";
+ for (const ref of [retained, removed, added])
+ await receipt(f.root, ref, "Shared evidence\n");
+ await recordCodexSecurityWorkerScanDraft(
+ f.worker,
+ draft([retained, removed]),
+ );
+ for (const ref of [retained, added])
+ await receipt(f.workerRoot, ref, "Local evidence\n");
+ const result = await recordCodexSecurityWorkerScanDraft(
+ f.worker,
+ draft([retained, added], { notes: "Updated review." }),
+ );
+ assert.deepEqual(result.coverage.surfaces[0].provenance.scanReceiptRefs, [
+ retained,
+ ]);
+ const sources = await readDeepReductionSources(f.reducer);
+ assert.deepEqual(sources.discoveries[0].coverage.surfaces[0].receiptRefs, [
+ retained,
+ `${f.prefix}/${added}`,
+ ]);
+ } finally {
+ await rm(f.root, { recursive: true, force: true });
+ }
+});
+
+test("ambiguous saved surface IDs do not transfer shared ownership to an edited observation", async () => {
+ const f = await fixture();
+ try {
+ const ref = "artifacts/evidence.txt";
+ await receipt(f.root, ref, "Shared evidence\n");
+ await recordCodexSecurityWorkerScanDraft(f.worker, draft([ref]));
+ const resultPath = path.join(f.workerRoot, "result.json");
+ const previous = JSON.parse(await readFile(resultPath, "utf8"));
+ previous.coverage.surfaces.push({
+ ...previous.coverage.surfaces[0],
+ label: "Separate review with a reused ID",
+ });
+ await writeFile(resultPath, JSON.stringify(previous));
+ await receipt(f.workerRoot, ref, "Local evidence\n");
+ await recordCodexSecurityWorkerScanDraft(
+ f.worker,
+ draft([ref], { notes: "Updated review." }),
+ );
+ const sources = await readDeepReductionSources(f.reducer);
+ const edited = sources.discoveries[0].coverage.surfaces.find(
+ (surface: { notes?: string }) => surface.notes === "Updated review.",
+ );
+ assert.deepEqual(edited.receiptRefs, [`${f.prefix}/${ref}`]);
+ } finally {
+ await rm(f.root, { recursive: true, force: true });
+ }
+});
+
+for (const archived of [false, true]) {
+ test(`an unrecorded ${archived ? "archived" : "current"} result cannot transfer shared ownership through a later draft`, async () => {
+ const f = await fixture();
+ try {
+ const ref = "artifacts/evidence.txt";
+ await receipt(f.root, ref, "Unrelated shared evidence\n");
+ await receipt(f.workerRoot, ref, "Local evidence\n");
+ await writeFile(
+ path.join(f.workerRoot, "result.json"),
+ JSON.stringify(
+ draft([ref], { provenance: { scanReceiptRefs: [ref] } }),
+ ),
+ );
+ let prefix = f.prefix;
+ if (archived) {
+ const archive = path.join(
+ path.dirname(f.workerRoot),
+ "attempts/attempt-01",
+ );
+ await mkdir(path.dirname(archive), { recursive: true });
+ await rename(f.workerRoot, archive);
+ await mkdir(f.workerRoot);
+ prefix = `${f.prefix}/artifacts/archived-receipts/attempt-01`;
+ }
+ await recordCodexSecurityWorkerScanDraft(
+ f.worker,
+ archived ? workerDraft([]) : draft([ref], { notes: "Updated review." }),
+ );
+ const sources = await readDeepReductionSources(f.reducer);
+ const [surface] = sources.discoveries[0].coverage.surfaces;
+ assert.deepEqual(surface.receiptRefs, [`${prefix}/${ref}`]);
+ assert.equal(
+ await readFile(path.join(f.root, surface.receiptRefs[0]), "utf8"),
+ "Local evidence\n",
+ );
+ } finally {
+ await rm(f.root, { recursive: true, force: true });
+ }
+ });
+}
+
+test("newly resolved shared receipts survive a later local collision alongside inherited receipts", async () => {
+ const f = await fixture();
+ try {
+ const refs = ["artifacts/first.txt", "artifacts/second.txt"];
+ for (const ref of refs) await receipt(f.root, ref, "Shared evidence\n");
+ await recordCodexSecurityWorkerScanDraft(f.worker, draft([refs[0]!]));
+ await recordCodexSecurityWorkerScanDraft(f.worker, draft(refs));
+ await receipt(f.workerRoot, refs[1]!, "Unrelated local evidence\n");
+ const sources = await readDeepReductionSources(f.reducer);
+ assert.deepEqual(
+ sources.discoveries[0].coverage.surfaces[0].receiptRefs,
+ refs,
+ );
+ } finally {
+ await rm(f.root, { recursive: true, force: true });
+ }
+});
+
+for (const complete of [false, true]) {
+ for (const archived of [false, true]) {
+ for (const lost of [false, true]) {
+ test(`first saved local receipt in a ${complete ? "result" : "checkpoint"} stays owned after ${archived ? "archiving" : "acceptance"} (${lost ? "lost" : "present"})`, async () => {
+ const f = await fixture();
+ try {
+ const ref = "artifacts/evidence.txt";
+ await receipt(f.workerRoot, ref, "Local evidence\n");
+ await receipt(f.root, ref, "Unrelated shared evidence\n");
+ await recordCodexSecurityWorkerScanDraft(f.worker, {
+ ...draft([ref]),
+ complete,
+ });
+ let output = f.workerRoot;
+ if (archived) {
+ output = path.join(
+ path.dirname(f.workerRoot),
+ "attempts/attempt-01",
+ );
+ await mkdir(path.dirname(output), { recursive: true });
+ await rename(f.workerRoot, output);
+ }
+ if (lost) await rm(path.join(output, ref));
+ if (archived) {
+ await mkdir(f.workerRoot);
+ await recordCodexSecurityWorkerScanDraft(f.worker, workerDraft([]));
+ }
+ const expected =
+ archived && !lost
+ ? `${f.prefix}/artifacts/archived-receipts/attempt-01/${ref}`
+ : `${path.relative(f.root, output).split(path.sep).join("/")}/${ref}`;
+ if (complete) {
+ const sources = await readDeepReductionSources(f.reducer);
+ assert.deepEqual(
+ sources.discoveries[0].coverage.surfaces[0].receiptRefs,
+ [expected],
+ );
+ }
+ const recovered = recoverCoverage(f, complete);
+ assert.deepEqual(recovered.refs, [expected]);
+ assert.equal(
+ recovered.surfaces[0].disposition,
+ lost ? "needs_follow_up" : "no_issue_found",
+ );
+ assert.equal(
+ recovered.warnings.some((warning: string) =>
+ warning.includes("coverage receipt"),
+ ),
+ lost,
+ );
+ } finally {
+ await rm(f.root, { recursive: true, force: true });
+ }
+ });
+ }
+ }
+}
+
+test("a draft preserved after a malformed result keeps its local receipt after loss", async () => {
+ const f = await fixture();
+ try {
+ const ref = "artifacts/evidence.txt";
+ await receipt(f.workerRoot, ref, "Local evidence\n");
+ await receipt(f.root, ref, "Unrelated shared evidence\n");
+ const resultPath = path.join(f.workerRoot, "result.json");
+ await writeFile(resultPath, "{broken");
+ await assert.rejects(
+ recordCodexSecurityWorkerScanDraft(f.worker, draft([ref])),
+ /stored JSON is malformed/,
+ );
+ await rm(resultPath);
+ await rm(path.join(f.workerRoot, ref));
+ await recordCodexSecurityWorkerScanDraft(f.worker, workerDraft([]));
+ const sources = await readDeepReductionSources(f.reducer);
+ assert.deepEqual(sources.discoveries[0].coverage.surfaces[0].receiptRefs, [
+ `${f.prefix}/${ref}`,
+ ]);
+ } finally {
+ await rm(f.root, { recursive: true, force: true });
+ }
+});
+
+for (const spelling of ["original", "returned"] as const) {
+ for (const sharedExists of [false, true]) {
+ for (const observation of [
+ "same",
+ "notes",
+ "new id",
+ "new disposition",
+ ] as const) {
+ for (const newLocal of [false, true]) {
+ test(`resubmitted ${spelling} receipt survives two archives: ${observation}, shared=${sharedExists}, new local=${newLocal}`, async () => {
+ const f = await fixture();
+ try {
+ const ref = "artifacts/evidence.txt";
+ await receipt(f.workerRoot, ref, "Original local evidence\n");
+ if (sharedExists)
+ await receipt(f.root, ref, "Unrelated shared evidence\n");
+ const first = await recordCodexSecurityWorkerScanDraft(
+ f.worker,
+ draft([ref]),
+ );
+ const submitted =
+ spelling === "original"
+ ? ref
+ : first.coverage.surfaces[0].receiptRefs[0];
+ let ownedRef = `${f.prefix}/${ref}`;
+ let contents = "Original local evidence\n";
+ for (const attempt of [1, 2]) {
+ const archive = path.join(
+ path.dirname(f.workerRoot),
+ "attempts",
+ `attempt-0${attempt}`,
+ );
+ const priorResult = await readFile(
+ path.join(f.workerRoot, "result.json"),
+ "utf8",
+ );
+ await archiveDirectory(f.workerRoot, archive);
+ if (ownedRef.startsWith(`${f.prefix}/`))
+ ownedRef = `${f.prefix}/artifacts/archived-receipts/attempt-0${attempt}/${ownedRef.slice(f.prefix.length + 1)}`;
+ const retained =
+ observation === "same" || observation === "notes";
+ if (newLocal) {
+ contents = `New local evidence ${attempt}\n`;
+ await receipt(f.workerRoot, ref, contents);
+ ownedRef = `${f.prefix}/${ref}`;
+ } else if (!retained) {
+ ownedRef =
+ sharedExists && spelling === "original"
+ ? ref
+ : `${f.prefix}/${ref}`;
+ contents = "Unrelated shared evidence\n";
+ }
+ const id =
+ observation === "new id"
+ ? `new-route-${attempt}`
+ : "reviewed-route";
+ await recordCodexSecurityWorkerScanDraft(
+ f.worker,
+ draft([submitted], {
+ id,
+ ...(observation === "notes"
+ ? { notes: `Retry notes ${attempt}` }
+ : {}),
+ ...(observation === "new disposition"
+ ? {
+ disposition:
+ attempt === 1 ? "not_applicable" : "rejected",
+ }
+ : {}),
+ }),
+ );
+ const sources = await readDeepReductionSources(f.reducer);
+ const projected = sources.discoveries[0].coverage.surfaces.find(
+ (surface: { provenance?: { sourceId?: string } }) =>
+ surface.provenance?.sourceId === id,
+ );
+ assert.ok(projected);
+ assert.deepEqual(projected.receiptRefs, [ownedRef]);
+ if (
+ newLocal ||
+ retained ||
+ (sharedExists && spelling === "original")
+ ) {
+ assert.equal(
+ await readFile(path.join(f.root, ownedRef), "utf8"),
+ contents,
+ );
+ } else {
+ await assert.rejects(readFile(path.join(f.root, ownedRef)), {
+ code: "ENOENT",
+ });
+ }
+ assert.equal(
+ await readFile(path.join(archive, "result.json"), "utf8"),
+ priorResult,
+ );
+ }
+ if (observation === "same" || observation === "notes") {
+ const recovered = recoverCoverage(f, true, 3);
+ assert.deepEqual(recovered.refs, [ownedRef]);
+ assert.equal(recovered.surfaces[0].disposition, "no_issue_found");
+ assert.deepEqual(recovered.warnings, []);
+ }
+ } finally {
+ await rm(f.root, { recursive: true, force: true });
+ }
+ });
+ }
+ }
+ }
+}
+
+for (const spelling of ["original", "returned"] as const) {
+ test(`resubmitted ${spelling} receipt cannot fall back to shared evidence after archived evidence is lost`, async () => {
+ const f = await fixture();
+ try {
+ const ref = "artifacts/evidence.txt";
+ await receipt(f.workerRoot, ref, "Original local evidence\n");
+ await receipt(f.root, ref, "Unrelated shared evidence\n");
+ const first = await recordCodexSecurityWorkerScanDraft(
+ f.worker,
+ draft([ref]),
+ );
+ const archive = path.join(
+ path.dirname(f.workerRoot),
+ "attempts/attempt-01",
+ );
+ await archiveDirectory(f.workerRoot, archive);
+ await rm(path.join(archive, ref));
+ await recordCodexSecurityWorkerScanDraft(
+ f.worker,
+ draft([
+ spelling === "original"
+ ? ref
+ : first.coverage.surfaces[0].receiptRefs[0],
+ ]),
+ );
+ const expected = `${path.relative(f.root, archive).split(path.sep).join("/")}/${ref}`;
+ const sources = await readDeepReductionSources(f.reducer);
+ assert.deepEqual(
+ sources.discoveries[0].coverage.surfaces[0].receiptRefs,
+ [expected],
+ );
+ const recovered = recoverCoverage(f, true, 2);
+ assert.deepEqual(recovered.refs, [expected]);
+ assert.equal(recovered.surfaces[0].disposition, "needs_follow_up");
+ assert.ok(
+ recovered.warnings.some((warning: string) =>
+ warning.includes("coverage receipt"),
+ ),
+ );
+ } finally {
+ await rm(f.root, { recursive: true, force: true });
+ }
+ });
+}
+
+for (const disposition of ["rejected", "not_applicable"]) {
+ for (const archived of [false, true]) {
+ test(`a ${disposition} decision cycle resolves fresh receipts (archived=${archived})`, async () => {
+ const f = await fixture();
+ try {
+ const ref = "artifacts/evidence.txt";
+ const candidateId = "candidate-1";
+ await receipt(f.root, ref, "Old shared evidence\n");
+ await recordCodexSecurityWorkerScanDraft(
+ f.worker,
+ draft([ref], { disposition, candidateId }),
+ );
+ for (const [index, next] of [
+ "needs_follow_up",
+ disposition,
+ ].entries()) {
+ if (archived)
+ await archiveDirectory(
+ f.workerRoot,
+ path.join(
+ path.dirname(f.workerRoot),
+ "attempts",
+ `attempt-0${index + 1}`,
+ ),
+ );
+ await receipt(
+ f.workerRoot,
+ ref,
+ index ? "New final local evidence\n" : "Reopened local evidence\n",
+ );
+ const input = draft([ref], { disposition: next, candidateId });
+ const result = await recordCodexSecurityWorkerScanDraft(f.worker, {
+ ...input,
+ coverage: {
+ ...input.coverage,
+ ...(next === "needs_follow_up"
+ ? {
+ completeness: "partial",
+ deferred: [
+ {
+ id: candidateId,
+ candidateId,
+ reason: "Revalidate the candidate.",
+ },
+ ],
+ }
+ : {}),
+ },
+ });
+ assert.deepEqual(result.coverage.surfaces[0].receiptRefs, [
+ `${f.prefix}/${ref}`,
+ ]);
+ }
+ for (const reconstructed of [false, true]) {
+ if (reconstructed) {
+ await rm(path.join(f.workerRoot, "result.json"));
+ await recordCodexSecurityWorkerScanDraft(f.worker, workerDraft([]));
+ }
+ const sources = await readDeepReductionSources(f.reducer);
+ const coverage = sources.discoveries[0].coverage;
+ assert.deepEqual(coverage.deferred, []);
+ assert.equal(coverage.surfaces.length, 1);
+ assert.equal(coverage.surfaces[0].disposition, disposition);
+ assert.deepEqual(coverage.surfaces[0].receiptRefs, [
+ `${f.prefix}/${ref}`,
+ ]);
+ assert.equal(
+ await readFile(
+ path.join(f.root, coverage.surfaces[0].receiptRefs[0]),
+ "utf8",
+ ),
+ "New final local evidence\n",
+ );
+ const recovered = recoverCoverage(f, true, archived ? 3 : 1);
+ assert.deepEqual(recovered.refs, [`${f.prefix}/${ref}`]);
+ assert.equal(recovered.surfaces[0].disposition, disposition);
+ assert.deepEqual(recovered.warnings, []);
+ }
+ } finally {
+ await rm(f.root, { recursive: true, force: true });
+ }
+ });
+ }
+}
diff --git a/plugins/codex-security/mcp-app/tests/test_retry_coverage_dispositions.ts b/plugins/codex-security/mcp-app/tests/test_retry_coverage_dispositions.ts
new file mode 100644
index 000000000..594b8f922
--- /dev/null
+++ b/plugins/codex-security/mcp-app/tests/test_retry_coverage_dispositions.ts
@@ -0,0 +1,150 @@
+import assert from "node:assert/strict";
+import { mkdtemp, readFile, rm } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import path from "node:path";
+import test from "node:test";
+import {
+ publishCoverageFixture,
+ readFixtureCoverage,
+} from "./deep_scan_coverage_fixture.ts";
+
+for (const surfaceOnly of [false, true]) {
+ for (const resume of [false, true]) {
+ for (const disposition of ["rejected", "not_applicable", "reported"]) {
+ for (const resolveOnRetry of [false, true]) {
+ test(`${resume ? "resumed" : "live"} ${surfaceOnly ? "surface-only" : "deferred"} stopped recovery preserves ${resolveOnRetry ? "the later disposition" : "newer gaps"} after a ${disposition} retry`, async () => {
+ const root = await mkdtemp(path.join(tmpdir(), "retry-disposition-"));
+ try {
+ const resolved = {
+ completeness: "complete",
+ surfaces: [
+ {
+ id: "disposition",
+ label: "Candidate disposition",
+ candidateId: "candidate-1",
+ disposition,
+ receiptRefs: [],
+ },
+ ],
+ explicitExclusions: [],
+ deferred: [],
+ openQuestions: [],
+ };
+ const pending = {
+ completeness: "partial",
+ surfaces: [
+ {
+ id: "follow-up",
+ label: "Retry boundary review",
+ candidateId: "candidate-1",
+ disposition: "needs_follow_up",
+ receiptRefs: [],
+ },
+ ],
+ explicitExclusions: [],
+ deferred: [
+ "First boundary needs proof.",
+ "Second boundary needs proof.",
+ ].map((reason, index) => ({
+ id: `retry-gap-${index}`,
+ candidateId: "candidate-1",
+ reason,
+ surfaceIds: ["follow-up"],
+ })),
+ openQuestions: [],
+ };
+ if (surfaceOnly) {
+ pending.deferred = [];
+ pending.surfaces.push({
+ ...pending.surfaces[0]!,
+ id: "follow-up-second",
+ });
+ }
+ const findings =
+ disposition === "reported"
+ ? [
+ {
+ ruleId: "synthetic.retained-boundary",
+ title: "Synthetic boundary finding",
+ summary: "A finding retained from an earlier review.",
+ severity: {
+ level: "high",
+ score: 8.1,
+ scoringSystem: "CVSS:3.1",
+ },
+ confidence: {
+ level: "high",
+ rationale: "Synthetic evidence.",
+ },
+ taxonomy: { category: "path-traversal", cwe: ["CWE-22"] },
+ locations: [
+ { path: "source.py", startLine: 1, endLine: 1 },
+ ],
+ remediation: "Review the synthetic boundary.",
+ provenance: {
+ source: "local_plugin",
+ candidateId: "candidate-1",
+ },
+ },
+ ]
+ : undefined;
+ const { scanDir } = await publishCoverageFixture(root, "partial", {
+ stopBeforeDraft: true,
+ resume,
+ retryCoverage: resolveOnRetry
+ ? [pending, resolved]
+ : [resolved, pending],
+ retryFindings:
+ findings && (resolveOnRetry ? [[], findings] : [findings, []]),
+ });
+ const coverage = await readFixtureCoverage(scanDir);
+ const retryGaps = coverage.deferred.filter((item) =>
+ pending.deferred.some((gap) => gap.reason === item.reason),
+ );
+ if (findings) {
+ const retained = JSON.parse(
+ await readFile(path.join(scanDir, "findings.json"), "utf8"),
+ );
+ assert.equal(retained.findings.length, 1);
+ assert.equal(retained.findings[0].ruleId, findings[0].ruleId);
+ }
+ assert.deepEqual(
+ retryGaps.map((item) => item.reason),
+ resolveOnRetry ? [] : pending.deferred.map((item) => item.reason),
+ );
+ assert.equal(
+ coverage.surfaces.filter(
+ (item) => item.label === "Retry boundary review",
+ ).length,
+ resolveOnRetry ? 0 : pending.surfaces.length,
+ );
+ assert.equal(
+ coverage.surfaces.filter(
+ (item) => item.disposition === disposition,
+ ).length,
+ resolveOnRetry || (surfaceOnly && disposition !== "reported")
+ ? 1
+ : 0,
+ );
+ assert.ok(
+ coverage.deferred.some(
+ (item) => item.reason === "Verify symbolic links.",
+ ),
+ "another worker's same-candidate gap remains independent",
+ );
+ for (const gap of retryGaps) {
+ assert.equal(
+ coverage.surfaces.find(
+ (surface) => surface.id === gap.surfaceIds[0],
+ )?.label,
+ "Retry boundary review",
+ );
+ }
+ } finally {
+ await rm(root, { recursive: true, force: true });
+ }
+ });
+ }
+ }
+ }
+}
diff --git a/plugins/codex-security/mcp-app/tests/test_retry_coverage_receipts.ts b/plugins/codex-security/mcp-app/tests/test_retry_coverage_receipts.ts
new file mode 100644
index 000000000..f0e958465
--- /dev/null
+++ b/plugins/codex-security/mcp-app/tests/test_retry_coverage_receipts.ts
@@ -0,0 +1,394 @@
+import { mkdtemp, rm } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import path from "node:path";
+import test from "node:test";
+import { publishCoverageFixture } from "./deep_scan_coverage_fixture.ts";
+
+for (const resume of [false, true]) {
+ for (const outcome of ["completion", "recovery", "no parent"]) {
+ for (const retryPending of [false, true]) {
+ test(`archived receipt and pending records survive ${resume ? "reconstruction" : "live retry"} and ${outcome} with ${retryPending ? "an updated gap" : "a clean retry"}`, async () => {
+ const root = await mkdtemp(path.join(tmpdir(), "retry-coverage-"));
+ try {
+ await publishCoverageFixture(root, "complete", {
+ receiptRetry: true,
+ retryPending,
+ stopAfterDraft: outcome === "recovery",
+ stopBeforeDraft: outcome === "no parent",
+ resume,
+ });
+ } finally {
+ await rm(root, { recursive: true, force: true });
+ }
+ });
+ }
+ }
+}
+
+for (const resume of [false, true]) {
+ for (const outcome of ["completion", "recovery", "no parent"]) {
+ for (const receiptSpelling of ["scan", "equivalent scan"] as const) {
+ test(`receipt namespace survives ${resume ? "reconstruction" : "live retry"} and ${outcome} with ${receiptSpelling} spelling`, async () => {
+ const root = await mkdtemp(path.join(tmpdir(), "receipt-namespace-"));
+ try {
+ await publishCoverageFixture(root, "complete", {
+ receiptRetry: true,
+ receiptSpelling,
+ stopAfterDraft: outcome === "recovery",
+ stopBeforeDraft: outcome === "no parent",
+ resume,
+ });
+ } finally {
+ await rm(root, { recursive: true, force: true });
+ }
+ });
+ }
+ }
+}
+
+for (const resume of [false, true]) {
+ for (const outcome of ["completion", "recovery", "no parent"]) {
+ for (const activeReceiptSpelling of ["scan", "equivalent scan"] as const) {
+ test(`active receipt namespace survives ${resume ? "reconstruction" : "live retry"} and ${outcome} with ${activeReceiptSpelling} spelling`, async () => {
+ const root = await mkdtemp(
+ path.join(tmpdir(), "active-receipt-namespace-"),
+ );
+ try {
+ await publishCoverageFixture(root, "complete", {
+ receiptRetry: true,
+ activeReceiptSpelling,
+ stopAfterDraft: outcome === "recovery",
+ stopBeforeDraft: outcome === "no parent",
+ resume,
+ });
+ } finally {
+ await rm(root, { recursive: true, force: true });
+ }
+ });
+ }
+ }
+}
+
+for (const resume of [false, true]) {
+ for (const outcome of ["completion", "recovery", "no parent"]) {
+ for (const receiptSpelling of [
+ "active scan",
+ "equivalent active scan",
+ ] as const) {
+ test(`linked archived receipts survive ${resume ? "reconstruction" : "live retry"} and ${outcome} with ${receiptSpelling} spelling`, async () => {
+ const root = await mkdtemp(path.join(tmpdir(), "receipt-namespace-"));
+ try {
+ await publishCoverageFixture(root, "complete", {
+ receiptRetry: true,
+ receiptSpelling,
+ stopAfterDraft: outcome === "recovery",
+ stopBeforeDraft: outcome === "no parent",
+ resume,
+ });
+ } finally {
+ await rm(root, { recursive: true, force: true });
+ }
+ });
+ }
+ }
+}
+
+for (const resume of [false, true]) {
+ for (const outcome of ["completion", "recovery", "no parent"]) {
+ test(`shared scan receipts survive ${resume ? "reconstruction" : "live retry"} and ${outcome}`, async () => {
+ const root = await mkdtemp(path.join(tmpdir(), "shared-scan-receipts-"));
+ try {
+ await publishCoverageFixture(root, "complete", {
+ receiptRetry: true,
+ sharedReceipt: true,
+ stopAfterDraft: outcome === "recovery",
+ stopBeforeDraft: outcome === "no parent",
+ resume,
+ });
+ } finally {
+ await rm(root, { recursive: true, force: true });
+ }
+ });
+ }
+}
+
+for (const resume of [false, true]) {
+ for (const outcome of ["completion", "recovery", "no parent"]) {
+ for (const receiptSpelling of [
+ "shared scan",
+ "equivalent shared scan",
+ ] as const) {
+ test(`archived shared receipts survive ${resume ? "reconstruction" : "live retry"} and ${outcome} with ${receiptSpelling} spelling`, async () => {
+ const root = await mkdtemp(
+ path.join(tmpdir(), "archived-shared-receipts-"),
+ );
+ try {
+ await publishCoverageFixture(root, "complete", {
+ receiptRetry: true,
+ receiptSpelling,
+ stopAfterDraft: outcome === "recovery",
+ stopBeforeDraft: outcome === "no parent",
+ resume,
+ });
+ } finally {
+ await rm(root, { recursive: true, force: true });
+ }
+ });
+ }
+ }
+}
+
+for (const resume of [false, true]) {
+ for (const outcome of ["completion", "recovery", "no parent"]) {
+ for (const sharedReceipt of [false, true]) {
+ test(`empty receipt survives ${resume ? "reconstruction" : "live retry"} and ${outcome} with ${sharedReceipt ? "shared" : "worker"} current evidence`, async () => {
+ const root = await mkdtemp(
+ path.join(tmpdir(), "empty-current-receipts-"),
+ );
+ try {
+ await publishCoverageFixture(root, "complete", {
+ receiptRetry: true,
+ sharedReceipt,
+ emptyReceipt: true,
+ stopAfterDraft: outcome === "recovery",
+ stopBeforeDraft: outcome === "no parent",
+ resume,
+ });
+ } finally {
+ await rm(root, { recursive: true, force: true });
+ }
+ });
+ }
+ for (const receiptSpelling of [
+ "shared scan",
+ "equivalent shared scan",
+ ] as const) {
+ test(`empty receipt survives ${resume ? "reconstruction" : "live retry"} and ${outcome} with archived ${receiptSpelling} evidence`, async () => {
+ const root = await mkdtemp(
+ path.join(tmpdir(), "empty-archived-receipts-"),
+ );
+ try {
+ await publishCoverageFixture(root, "complete", {
+ receiptRetry: true,
+ receiptSpelling,
+ emptyReceipt: true,
+ stopAfterDraft: outcome === "recovery",
+ stopBeforeDraft: outcome === "no parent",
+ resume,
+ });
+ } finally {
+ await rm(root, { recursive: true, force: true });
+ }
+ });
+ }
+ }
+}
+
+for (const resume of [false, true]) {
+ for (const outcome of ["completion", "recovery", "no parent"]) {
+ test(`worker-local receipts win parent collisions during ${resume ? "reconstruction" : "live retry"} and ${outcome}`, async () => {
+ const root = await mkdtemp(
+ path.join(tmpdir(), "worker-receipt-collision-"),
+ );
+ try {
+ await publishCoverageFixture(root, "complete", {
+ receiptRetry: true,
+ receiptCollision: true,
+ stopAfterDraft: outcome === "recovery",
+ stopBeforeDraft: outcome === "no parent",
+ resume,
+ });
+ } finally {
+ await rm(root, { recursive: true, force: true });
+ }
+ });
+ }
+}
+
+for (const resume of [false, true]) {
+ for (const outcome of ["completion", "recovery", "no parent"]) {
+ test(`inherited shared receipts retain their owner across retry collisions during ${resume ? "reconstruction" : "live retry"} and ${outcome}`, async () => {
+ const root = await mkdtemp(
+ path.join(tmpdir(), "shared-receipt-collision-"),
+ );
+ try {
+ await publishCoverageFixture(root, "complete", {
+ receiptRetry: true,
+ receiptSpelling: "shared scan",
+ receiptCollision: true,
+ stopAfterDraft: outcome === "recovery",
+ stopBeforeDraft: outcome === "no parent",
+ resume,
+ });
+ } finally {
+ await rm(root, { recursive: true, force: true });
+ }
+ });
+ }
+}
+
+const receiptOwnershipCases = [
+ [
+ "third attempt shared collision",
+ {
+ extraReceiptRetry: true,
+ receiptSpelling: "shared scan",
+ receiptCollision: true,
+ },
+ ],
+ [
+ "returned shared coverage rewrite collision",
+ {
+ rewriteReturnedCoverage: true,
+ receiptSpelling: "shared scan",
+ receiptCollision: true,
+ },
+ ],
+ [
+ "shared and new local same spelling",
+ {
+ sameNamedNewSurface: true,
+ receiptSpelling: "shared scan",
+ receiptCollision: true,
+ },
+ ],
+ [
+ "third attempt shared no collision control",
+ {
+ extraReceiptRetry: true,
+ receiptSpelling: "shared scan",
+ receiptCollision: false,
+ },
+ ],
+ [
+ "returned worker-local coverage control",
+ {
+ rewriteReturnedCoverage: true,
+ receiptSpelling: "worker",
+ receiptCollision: true,
+ },
+ ],
+ [
+ "worker-local same spelling control",
+ {
+ sameNamedNewSurface: true,
+ receiptSpelling: "worker",
+ receiptCollision: true,
+ },
+ ],
+] as const;
+for (const [name, options] of receiptOwnershipCases)
+ test(name, async () => {
+ const root = await mkdtemp(path.join(tmpdir(), "supported-receipt-owner-"));
+ try {
+ await publishCoverageFixture(root, "complete", {
+ receiptRetry: true,
+ ...options,
+ });
+ } finally {
+ await rm(root, { recursive: true, force: true });
+ }
+ });
+
+for (const resume of [false, true]) {
+ for (const [name, receiptSpelling, receiptCollision] of [
+ ["shared same-name collision", "shared scan", true],
+ ["shared distinct-name control", "shared scan", false],
+ ["worker-local same-name control", "worker", true],
+ ] as const) {
+ test(`same surface generic closeout retains both receipt owners: ${name}, ${resume ? "reconstructed" : "live"}`, async () => {
+ const root = await mkdtemp(path.join(tmpdir(), "same-surface-closeout-"));
+ try {
+ await publishCoverageFixture(root, "complete", {
+ receiptRetry: true,
+ closeRetriedSurface: true,
+ receiptSpelling,
+ receiptCollision,
+ resume,
+ });
+ } finally {
+ await rm(root, { recursive: true, force: true });
+ }
+ });
+ }
+}
+
+for (const resume of [false, true]) {
+ for (const receiptOwnershipRetry of [
+ "ordinary",
+ "candidate",
+ "unrelated",
+ ] as const) {
+ for (const [name, receiptSpelling, receiptCollision] of [
+ ["shared same-name collision", "shared scan", true],
+ ["shared distinct-name control", "shared scan", false],
+ ["worker-local control", "worker", true],
+ ] as const) {
+ test(`receipt ownership follows the authored observation: ${receiptOwnershipRetry}, ${name}, ${resume ? "reconstructed" : "live"}`, async () => {
+ const root = await mkdtemp(
+ path.join(tmpdir(), "receipt-owner-observation-"),
+ );
+ try {
+ await publishCoverageFixture(root, "complete", {
+ receiptRetry: true,
+ receiptOwnershipRetry,
+ receiptSpelling,
+ receiptCollision,
+ resume,
+ });
+ } finally {
+ await rm(root, { recursive: true, force: true });
+ }
+ });
+ }
+ }
+}
+
+for (const resume of [false, true]) {
+ for (const outcome of ["completion", "recovery", "no parent"]) {
+ for (const [name, firstCheckpointReceipt, receiptCollision] of [
+ ["shared same-name collision", "shared", true],
+ ["shared distinct-name control", "shared", false],
+ ["worker-local priority control", "worker", false],
+ ] as const) {
+ test(`first checkpoint receipt ownership survives ${resume ? "reconstruction" : "live execution"} and ${outcome}: ${name}`, async () => {
+ const root = await mkdtemp(
+ path.join(tmpdir(), "first-checkpoint-owner-"),
+ );
+ try {
+ await publishCoverageFixture(root, "complete", {
+ firstCheckpointReceipt,
+ receiptCollision,
+ resume,
+ stopAfterDraft: outcome === "recovery",
+ stopBeforeDraft: outcome === "no parent",
+ });
+ } finally {
+ await rm(root, { recursive: true, force: true });
+ }
+ });
+ }
+ }
+}
+
+for (const resume of [false, true]) {
+ for (const [name, firstCheckpointReceipt, receiptCollision] of [
+ ["shared same-name collision", "shared", true],
+ ["shared distinct-name control", "shared", false],
+ ["worker-local priority control", "worker", false],
+ ] as const) {
+ test(`same attempt closeout retains both receipt owners: ${name}, ${resume ? "reconstructed" : "live"}`, async () => {
+ const root = await mkdtemp(path.join(tmpdir(), "same-attempt-closeout-"));
+ try {
+ await publishCoverageFixture(root, "complete", {
+ firstCheckpointReceipt,
+ sameAttemptCloseout: true,
+ receiptCollision,
+ resume,
+ });
+ } finally {
+ await rm(root, { recursive: true, force: true });
+ }
+ });
+ }
+}
diff --git a/plugins/codex-security/mcp-app/tests/test_worker_draft_concurrency.ts b/plugins/codex-security/mcp-app/tests/test_worker_draft_concurrency.ts
new file mode 100644
index 000000000..bf26064cf
--- /dev/null
+++ b/plugins/codex-security/mcp-app/tests/test_worker_draft_concurrency.ts
@@ -0,0 +1,156 @@
+import assert from "node:assert/strict";
+import { promises as fs } from "node:fs";
+import path from "node:path";
+import test from "node:test";
+import { Client } from "@modelcontextprotocol/sdk/client/index.js";
+import { InMemoryTransport } from "@modelcontextprotocol/sdk/inMemory.js";
+import { importSource } from "./import-module.ts";
+import { finding, scanId, workerDraft } from "./scan-draft-fixture.ts";
+import { temporaryDirectory } from "./support/temporary-directories.ts";
+
+const { createCodexSecurityArtifactWriterServer } = await importSource(
+ path.join(import.meta.dirname, "../artifact-writer-main.ts"),
+);
+const { readDeepReductionSources } = await importSource(
+ path.join(import.meta.dirname, "../src/artifact-deep-reducer.ts"),
+);
+
+async function worker(root: string, label: string) {
+ const output = path.join(
+ root,
+ "artifacts/deep_discovery/workers",
+ label,
+ "output",
+ );
+ const reducerRoot = path.join(
+ root,
+ "artifacts/deep_discovery/dedup/run/output",
+ );
+ await fs.mkdir(output, { recursive: true });
+ await fs.mkdir(reducerRoot, { recursive: true });
+ const server = await createCodexSecurityArtifactWriterServer({
+ CODEX_SECURITY_ARTIFACT_ROOT: output,
+ CODEX_SECURITY_REPO_ROOT: root,
+ CODEX_SECURITY_SCAN_ID: scanId,
+ });
+ const client = new Client({ name: "worker-draft-fixture", version: "1.0.0" });
+ const [clientTransport, serverTransport] =
+ InMemoryTransport.createLinkedPair();
+ await server.connect(serverTransport);
+ await client.connect(clientTransport);
+ return {
+ output,
+ draft: (id: string) =>
+ client.callTool({
+ name: "record_codex_security_scan_draft",
+ arguments: workerDraft([finding(id, `src/${id}.ts`)]),
+ }),
+ async findings() {
+ const sources = await readDeepReductionSources({
+ root: reducerRoot,
+ repoRoot: root,
+ scanId,
+ layout: "reducer",
+ deepReducer: {
+ scanRoot: root,
+ claimedWorkers: [
+ { id: label, resultPath: path.join(output, "result.json") },
+ ],
+ },
+ });
+ return sources.discoveries[0].result.findings
+ .map((row: { identity: { anchor: string } }) => row.identity.anchor)
+ .sort();
+ },
+ async close() {
+ await client.close();
+ await server.close();
+ },
+ };
+}
+
+test("concurrent registered worker drafts retain both findings for reduction", async () => {
+ const root = await temporaryDirectory("worker-draft-concurrent-", true);
+ const w = await worker(root, "discovery-0001");
+ try {
+ const replies = await Promise.all([w.draft("first"), w.draft("second")]);
+ for (const reply of replies) {
+ assert.equal(reply.isError, undefined);
+ assert.equal(
+ JSON.parse((reply.content as { text: string }[])[0]!.text).status,
+ "draft_written",
+ );
+ }
+ assert.deepEqual(await w.findings(), ["first", "second"]);
+ } finally {
+ await w.close();
+ await fs.rm(root, { recursive: true, force: true });
+ }
+});
+
+test("a blocked worker draft does not block another worker", async (t) => {
+ const root = await temporaryDirectory("worker-draft-independent-", true);
+ const first = await worker(root, "discovery-0001");
+ const other = await worker(root, "discovery-0002");
+ const blocked = Promise.withResolvers<void>();
+ const release = Promise.withResolvers<void>();
+ const rename = fs.rename;
+ let paused = false;
+ t.mock.method(fs, "rename", async (...args: Parameters<typeof rename>) => {
+ if (
+ !paused &&
+ String(args[1]).startsWith(
+ path.join(first.output, "checkpoints") + path.sep,
+ )
+ ) {
+ paused = true;
+ blocked.resolve();
+ await release.promise;
+ }
+ return rename(...args);
+ });
+ const pending = first.draft("first");
+ try {
+ await blocked.promise;
+ const reply = await other.draft("other");
+ assert.equal(reply.isError, undefined);
+ assert.deepEqual(await other.findings(), ["other"]);
+ release.resolve();
+ assert.equal((await pending).isError, undefined);
+ assert.deepEqual(await first.findings(), ["first"]);
+ } finally {
+ release.resolve();
+ await pending;
+ t.mock.restoreAll();
+ await Promise.all([first.close(), other.close()]);
+ await fs.rm(root, { recursive: true, force: true });
+ }
+});
+
+test("a failed worker write releases the next registered draft", async (t) => {
+ const root = await temporaryDirectory("worker-draft-failure-", true);
+ const w = await worker(root, "discovery-0001");
+ const rename = fs.rename;
+ let failed = false;
+ t.mock.method(fs, "rename", async (...args: Parameters<typeof rename>) => {
+ if (
+ !failed &&
+ String(args[1]).startsWith(path.join(w.output, "checkpoints") + path.sep)
+ ) {
+ failed = true;
+ throw new Error("Injected checkpoint write failure");
+ }
+ return rename(...args);
+ });
+ try {
+ const ids = ["first", "second"];
+ const replies = await Promise.all(ids.map((id) => w.draft(id)));
+ assert.equal(replies.filter((reply) => reply.isError).length, 1);
+ const accepted = replies.findIndex((reply) => !reply.isError);
+ assert.deepEqual(await w.findings(), [ids[accepted]]);
+ } finally {
+ t.mock.restoreAll();
+ await w.close();
+ await fs.rm(root, { recursive: true, force: true });
+ }
+});
diff --git a/plugins/codex-security/plugin-files.json b/plugins/codex-security/plugin-files.json
index 146d8899a..3501bfa12 100644
--- a/plugins/codex-security/plugin-files.json
+++ b/plugins/codex-security/plugin-files.json
@@ -31,6 +31,7 @@
"mcp/server.mjs",
"mcp/server.mjs.br.part-000",
"mcp/server.mjs.br.part-001",
+ "mcp/server.mjs.br.part-002",
"references/access-advisory.md",
"references/artifact-storage.md",
"references/core-scan.md",
@@ -80,6 +81,7 @@
"scripts/windows_scan_local_files.py",
"scripts/workbench/__init__.py",
"scripts/workbench/handoff.py",
+ "scripts/workbench/retained_coverage.py",
"scripts/workbench/runtime_toml.py",
"scripts/workbench/saved_coverage.py",
"scripts/workbench/saved_outputs.py",
@@ -87,6 +89,7 @@
"scripts/workbench/storage.py",
"scripts/workbench_cli.py",
"scripts/workbench_constants.py",
+ "scripts/workbench_coverage_receipts.py",
"scripts/workbench_db.py",
"scripts/workbench_feedback.py",
"scripts/workbench_finding_index.py",
@@ -97,7 +100,6 @@
"scripts/workbench_publication.py",
"scripts/workbench_remediation.py",
"scripts/workbench_saved_results.py",
- "scripts/workbench/retained_coverage.py",
"scripts/workbench_scan_history.py",
"scripts/workbench_scan_start.py",
"scripts/workbench_scan_usage.py",
@@ -130,6 +132,9 @@
"skills/propose-security-hardening/SKILL.md",
"skills/propose-security-hardening/agents/openai.yaml",
"skills/propose-security-hardening/references/proposal-format.md",
+ "skills/publish-findings/SKILL.md",
+ "skills/publish-findings/agents/openai.yaml",
+ "skills/publish-findings/references/wiz-import.md",
"skills/security-diff-scan/SKILL.md",
"skills/security-diff-scan/agents/openai.yaml",
"skills/security-scan/SKILL.md",
@@ -138,9 +143,6 @@
"skills/security-scan/references/scan-artifacts-and-ledger.md",
"skills/threat-model/SKILL.md",
"skills/threat-model/agents/openai.yaml",
- "skills/publish-findings/SKILL.md",
- "skills/publish-findings/agents/openai.yaml",
- "skills/publish-findings/references/wiz-import.md",
"skills/track-findings/SKILL.md",
"skills/track-findings/agents/openai.yaml",
"skills/track-findings/references/github-security-advisories.md",
diff --git a/plugins/codex-security/scripts/workbench/retained_coverage.py b/plugins/codex-security/scripts/workbench/retained_coverage.py
index f5f2a26c9..a45a125ab 100644
--- a/plugins/codex-security/scripts/workbench/retained_coverage.py
+++ b/plugins/codex-security/scripts/workbench/retained_coverage.py
@@ -3,56 +3,45 @@
from __future__ import annotations
import copy
+import hashlib
import os
import re
import unicodedata
-from collections import Counter
-from collections.abc import Callable, Iterator
-from pathlib import Path
+from collections import Counter, deque
+from collections.abc import Iterator
+from functools import partial
+from pathlib import Path, PurePosixPath
from typing import Any
from finalize_scan_contract import (
ContractError,
+ _finding_strength,
_has_identity_text,
_read_json,
- _require_portable_relative_path,
- _require_str,
- _validate_resolved_deferred,
- _validate_schema_node,
- finding_candidate_id,
- open_scan_local_file_descriptor,
-)
-from workbench.saved_coverage import (
- _project_worker_coverage_row,
- _saved_coverage_id,
-)
-from workbench.saved_sources import (
- _digest,
- _encoded,
-)
-
-_FINDING_IDENTITY_SCHEMA = _read_json(
- Path(__file__).resolve().parents[2] / "schemas" / "findings.schema.json"
-)["properties"]["findings"]["items"]["properties"]["identity"]
-
-
-import hashlib
-from pathlib import PurePosixPath
-
-from finalize_scan_contract import (
- _finding_strength,
_recover_unsealed_coverage,
_recover_unsealed_findings,
_require_safe_json_value,
_validate_current_findings,
+ _validate_schema_node,
+ finding_candidate_id,
write_scan_local_bytes,
)
-from workbench.saved_coverage import _prepare_worker_coverage_recovery
+from workbench.saved_coverage import (
+ _merge_tied_parent_observations,
+ _parent_observation_content,
+ _project_worker_closure,
+ _projected_worker_closure_sources,
+ _resolved_deferred_rows,
+ _saved_coverage_id,
+ prepare_saved_coverage_projection,
+)
from workbench.saved_sources import (
_capture_parent_observation,
_capture_saved_source,
_checkpoint_head_directory,
_checkpoint_paths,
+ _digest,
+ _encoded,
_freeze_source_times,
_frozen_source_times,
_is_source_order_snapshot,
@@ -62,8 +51,19 @@ from workbench.saved_sources import (
_read_saved_result,
_worker_outputs,
)
+from workbench_coverage_receipts import (
+ _deferred_candidate_id,
+ _generic_surface_updates,
+ receipt_checkpoints,
+ resolve_coverage_receipts,
+ retain_recorded_receipt_ownership,
+)
from workbench_target import committed_diff_snapshot_digest
-from workbench_validation import path_within_scope
+from workbench_validation import javascript_utf8, path_within_scope
+
+_FINDING_IDENTITY_SCHEMA = _read_json(
+ Path(__file__).resolve().parents[2] / "schemas" / "findings.schema.json"
+)["properties"]["findings"]["items"]["properties"]["identity"]
def _finding_identity(finding: dict[str, Any]) -> Any:
@@ -285,75 +285,6 @@ def _coverage_rows(coverage: dict[str, Any], field: str) -> list[Any]:
return rows if isinstance(rows, list) else []
-def _deferred_rows(coverage: dict[str, Any]) -> list[Any]:
- return _coverage_rows(coverage, "deferred")
-
-
-def _resolved_deferred_rows(draft: dict[str, Any], schema: dict[str, Any]) -> list[dict[str, Any]]:
- if draft.get("complete") is False:
- return []
- coverage = draft["coverage"]
- rows = coverage.get("resolvedDeferred", [])
- try:
- # Invalid closure metadata cannot discard the evidence it names.
- _validate_schema_node(rows, schema, "coverage.resolvedDeferred")
- _validate_resolved_deferred({**coverage, "deferred": _coverage_rows(coverage, "deferred")})
- except ContractError:
- return []
- return rows
-
-
-def _merge_tied_parent_observations(
- current: dict[str, Any], previous: dict[str, Any]
-) -> dict[str, Any]:
- merged = copy.deepcopy(current)
- for finding in previous["findings"]:
- if finding not in merged["findings"]:
- merged["findings"].append(copy.deepcopy(finding))
- coverage = merged["coverage"]
- for field in ("surfaces", "explicitExclusions", "deferred", "openQuestions"):
- output = coverage.setdefault(field, [])
- if isinstance(output, list):
- for row in _coverage_rows(previous["coverage"], field):
- if row not in output:
- output.append(copy.deepcopy(row))
- pending_ids = {
- identity
- for row in _coverage_rows(coverage, "deferred")
- if isinstance(row, dict)
- for identity in (row.get("id"), row.get("candidateId"))
- if isinstance(identity, str)
- }
- closure_schema = _read_json(
- Path(__file__).resolve().parents[2] / "schemas" / "coverage.schema.json"
- )["properties"]["resolvedDeferred"]
- closures = {}
- for observed in (current, previous):
- for row in _resolved_deferred_rows(observed, closure_schema):
- if row["id"] not in pending_ids:
- closures.setdefault(row["id"], copy.deepcopy(row))
- coverage.pop("resolvedDeferred", None)
- if closures:
- coverage["resolvedDeferred"] = list(closures.values())
- if current.get("complete") is False or previous.get("complete") is False:
- merged["complete"] = False
- coverage.pop("completeness", None)
- return merged
-
-
-def _deferred_candidate_id(
- row: dict[str, Any],
- owner: str | None,
- ambiguous_deferred: set[tuple[str | None, str]],
-) -> str | None:
- identity = row.get("candidateId") or row.get("id")
- if not isinstance(identity, str):
- return None
- if (owner, identity) in ambiguous_deferred and not _has_candidate_metadata(row):
- return None
- return identity
-
-
def _inherit_coverage_sources(
row: Any,
originals: list[Any],
@@ -372,268 +303,123 @@ def _inherit_coverage_sources(
paths.append(relative)
-def _generic_surface_updates(
- sources: list[tuple[str, dict[str, Any], str | None]],
- source_order: dict[str, tuple[int, int]],
- closed_deferred: dict[tuple[str | None, str], tuple[tuple[int, int], dict[str, Any], str]],
- active_deferred: dict[tuple[str | None, str], tuple[tuple[int, int], dict[str, Any], str]],
- resolved_candidates: dict[tuple[str | None, str], str],
- reopened_generic: set[tuple[str | None, str]],
- surface_schema: dict[str, Any],
- deferred_rows: dict[str, list[Any]],
- ambiguous_deferred: set[tuple[str | None, str]],
- canonical_rows: set[int],
- coverage_sources: dict[int, tuple[dict[str, Any], list[str]]],
- *,
- surface_is_retained: Callable[[str, str | None, dict[str, Any]], bool],
- eligible_receipt_sources: set[str],
- receipt_root: Path | None = None,
- receipt_warnings: list[str] | None = None,
- fallback_surfaces: list[dict[str, Any]] | None = None,
-) -> tuple[set[int], list[tuple[str | None, dict[str, Any], set[int]]]]:
- if not closed_deferred and not reopened_generic and receipt_root is None:
- return set(), []
-
- def receipt_error(ref: Any) -> ContractError | None:
- if receipt_root is None or not isinstance(ref, str):
- return ContractError("Saved coverage receipt is unavailable.")
+def _recover_preserved_worker_source(
+ scan_dir, relative, scan_id, binding, worker_id, worker, draft, warnings
+):
+ discarded_findings = []
+ # Recover imported records without relaxing canonical parent validation.
+ if "complete" in draft and not isinstance(draft["complete"], bool):
+ draft["complete"] = False
+ warnings.append(f"{relative}: Invalid complete marker; retained as a progress checkpoint.")
+ schema_dir = Path(__file__).resolve().parents[2] / "schemas"
+ if "threatModel" in draft:
try:
- descriptor = open_scan_local_file_descriptor(
- receipt_root, ref, "Saved coverage receipt"
+ model = draft["threatModel"]
+ if isinstance(model, dict):
+ model = {**model, "origin": "recovered"}
+ schema = _read_json(schema_dir / "scan-manifest.schema.json")
+ _validate_schema_node(
+ model,
+ schema["properties"]["scan"]["properties"]["threatModel"],
+ "threatModel",
+ schema,
)
- os.close(descriptor)
+ _require_safe_json_value(model, "threatModel")
except ContractError as exc:
- return exc
- return None
-
- def linked(row: dict[str, Any], identity: str | None) -> bool:
- surface_ids = row.get("surfaceIds", [])
- return identity is not None and (
- row.get("id") == identity or (isinstance(surface_ids, list) and identity in surface_ids)
+ draft.pop("threatModel")
+ warnings.append(f"{relative}: Skipped malformed threat model: {exc}.")
+ directory = Path(relative).parent
+ if directory.name == "checkpoints":
+ directory = directory.parent
+ recovered_findings = []
+ for index, finding in enumerate(draft["findings"]):
+ needs_identity = isinstance(finding, dict) and "identity" not in finding
+ _ensure_finding_identity(finding)
+ recovered = {"scanId": scan_id, "findings": [finding]}
+ source_warnings: list[str] = []
+ discarded = _recover_unsealed_findings(
+ {"scan": {"id": scan_id, "target": binding["target"]}},
+ recovered,
+ schema_dir,
+ scan_dir / directory,
+ source_warnings,
)
-
- saved_surfaces: dict[tuple[str | None, str], list[tuple[str, dict[str, Any]]]] = {}
- for relative, draft, owner in sources:
- for row in _coverage_rows(draft["coverage"], "surfaces"):
- if isinstance(row, dict) and isinstance(row.get("id"), str):
- saved_surfaces.setdefault((owner, row["id"]), []).append((relative, row))
- replaced: set[int] = set()
- updates: list[tuple[str | None, dict[str, Any], set[int]]] = []
- # Current selections across all surfaces take precedence over historical refs.
- receipt_paths: dict[str, str] = {}
- retained_surfaces: dict[tuple[str | None, str], dict[str, Any]] = {}
- if receipt_root is not None:
- receipt_surfaces: list[tuple[str | None, dict[str, Any]]] = []
- for key, matches in saved_surfaces.items():
- retained = [
- match for match in matches if surface_is_retained(match[0], key[0], match[1])
- ]
- if not retained:
- continue
- latest_source, latest_surface = max(retained, key=lambda match: source_order[match[0]])
- retained_surfaces[key] = latest_surface
- receipt_surfaces.extend(
- (key[0], row)
- for relative, row in retained
- if source_order[relative] == source_order[latest_source]
- )
- for relative, draft, owner in sources:
- for row in _coverage_rows(draft["coverage"], "surfaces"):
- if (
- isinstance(row, dict)
- and "id" not in row
- and surface_is_retained(relative, owner, row)
- ):
- receipt_surfaces.append((owner, row))
- receipt_surfaces.extend((None, row) for row in fallback_surfaces or [])
- for owner, latest_surface in receipt_surfaces:
- # Recovery repairs disposition and receipts before validating row
- # metadata. A row discarded there cannot reserve a receipt spelling.
- metadata = _project_worker_coverage_row(latest_surface, owner, "surfaces")
- metadata.setdefault("id", _saved_coverage_id(metadata))
- metadata.update(disposition="needs_follow_up", receiptRefs=[])
+ if recovered["findings"]:
try:
- _require_str(metadata, "id", "coverage.surfaces")
- _validate_schema_node(metadata, surface_schema, "coverage.surfaces")
- except ContractError:
- continue
- refs = latest_surface.get("receiptRefs", [])
- for ref in refs if isinstance(refs, list) else []:
- if receipt_error(ref) is None:
- normalized = _require_portable_relative_path(ref, "Saved coverage receipt")
- receipt_paths.setdefault(normalized.lower(), normalized)
- for relative, draft, owner in sources:
- closed_ids = {
- identity
- for (saved_owner, identity), (_, _, source) in closed_deferred.items()
- if saved_owner == owner and source == relative
- }
- reopened_ids = {
- identity
- for (saved_owner, identity), (_, _, source) in active_deferred.items()
- if saved_owner == owner and source == relative and (owner, identity) in reopened_generic
- }
- if not closed_ids and not reopened_ids and receipt_root is None:
- continue
- for surface in _coverage_rows(draft["coverage"], "surfaces"):
- if not isinstance(surface, dict):
- continue
- reopening = surface.get("disposition") == "needs_follow_up"
- work_ids = reopened_ids if reopening else closed_ids
- repair_only = not work_ids
- if repair_only and receipt_root is None:
- continue
- identity = surface.get("id")
- if not isinstance(identity, str):
- continue
- matches = saved_surfaces[(owner, identity)]
- # Receipt repair alone cannot close or reopen a saved surface.
- if repair_only and any(
- row.get("disposition") != surface.get("disposition") for _, row in matches
- ):
- continue
- # Older writers could assign one ID to distinct surfaces in a draft.
- # A closure cannot identify which of those observations it replaces.
- by_source = dict(matches)
- if any(row != by_source[saved_path] for saved_path, row in matches):
- continue
- if any(_has_candidate_metadata(row) for _, row in matches):
- continue
- if any(
- row is not surface
- and source_order[saved_path] >= source_order[relative]
- and row.get("disposition") != surface.get("disposition")
- for saved_path, row in matches
- ):
- continue
-
- if not reopening and any(
- saved_owner == owner
- and (
- not isinstance(row.get("id") or row.get("candidateId"), str)
- or (
- isinstance(row.get("id"), str)
- and (owner, row["id"]) in ambiguous_deferred
- and not _has_candidate_metadata(row)
- )
- )
- and linked(row, identity)
- for saved_path, _, saved_owner in sources
- for row in deferred_rows[saved_path]
- if isinstance(row, dict)
- ):
- continue
- if not reopening and any(
- saved_owner == owner
- and (owner, deferred_id) not in closed_deferred
- and (
- (candidate_id := _deferred_candidate_id(row, owner, ambiguous_deferred)) is None
- or (owner, candidate_id) not in resolved_candidates
+ _validate_current_findings(recovered, schema_dir)
+ except ContractError as exc:
+ warning = f"Skipped malformed finding: {exc}."
+ source_warnings.append(warning)
+ discarded.append(warning)
+ recovered["findings"] = []
+ if discarded:
+ discarded_findings.extend(discarded)
+ for recovered_finding in recovered["findings"]:
+ if needs_identity:
+ # Reconcile anonymous revisions before assigning sibling identities.
+ recovered_finding.pop("identity", None)
+ writeup = recovered_finding.pop("writeup", None)
+ if writeup is not None:
+ source_warnings.append(
+ "Retained worker writeup at "
+ f"{directory.as_posix()}/{writeup['reportPath']}; "
+ "the parent report renders this finding inline."
)
- and linked(row, identity)
- for (saved_owner, deferred_id), (_, row, _) in active_deferred.items()
- ):
- continue
- # An accepted checkpoint can update a saved surface by ID without
- # optional surfaceIds links on its generic task.
- receipt_matches = [
- (saved_path, row)
- for saved_path, row in matches
- if receipt_root is None
- or saved_path in eligible_receipt_sources
- or surface_is_retained(saved_path, owner, row)
- ]
- if not receipt_matches:
- continue
- latest_surface = max(receipt_matches, key=lambda match: source_order[match[0]])[1]
- target = retained_surfaces.get((owner, identity)) if repair_only else latest_surface
- if target is None:
+ recovered_findings.extend(recovered["findings"])
+ warnings.extend(f"{relative} finding {index + 1}: {warning}" for warning in source_warnings)
+ draft["findings"] = recovered_findings
+ for field in ("surfaces", "explicitExclusions", "deferred", "openQuestions"):
+ rows = draft["coverage"].get(field)
+ if not isinstance(rows, list):
+ continue
+ for index, row in enumerate(rows):
+ if field == "openQuestions" and isinstance(row, str):
+ row = rows[index] = {"question": row}
+ if not isinstance(row, dict):
continue
- update = copy.deepcopy(target)
- # Historical evidence can repair a retained row without promoting
- # an unselected observation's label, notes, or other metadata.
- refs = copy.deepcopy(latest_surface.get("receiptRefs", []))
- update["receiptRefs"] = refs
- replaced_receipt = False
- replacement_paths = dict(receipt_paths)
- if isinstance(refs, list):
- repaired = (
- not reopening and bool(refs) and all(receipt_error(ref) is None for ref in refs)
+ row["workerId"] = worker_id
+ if field == "surfaces":
+ row["receiptRefs"] = resolve_coverage_receipts(
+ scan_dir,
+ row,
+ worker,
+ relative,
)
- if repaired:
- for ref in refs:
- normalized = _require_portable_relative_path(ref, "Saved coverage receipt")
- key = normalized.lower()
- if key in replacement_paths and replacement_paths[key] != normalized:
- repaired = False
- break
- replacement_paths[key] = normalized
- for _, row in receipt_matches:
- previous_refs = row.get("receiptRefs", [])
- for ref in previous_refs if isinstance(previous_refs, list) else []:
- if ref not in refs:
- failure = receipt_error(ref) if repaired else None
- replaceable = failure is not None and isinstance(
- failure.__cause__, FileNotFoundError
- )
- if repaired and isinstance(ref, str):
- try:
- normalized = _require_portable_relative_path(
- ref, "Saved coverage receipt"
- )
- except ContractError as error:
- failure = error
- replaceable = True
- else:
- key = normalized.lower()
- previous = replacement_paths.get(key)
- if previous is not None and previous != normalized:
- failure = ContractError(
- f"Saved coverage receipt: duplicate artifact path: {previous} and {normalized}"
- )
- replaceable = True
- elif not replaceable:
- replacement_paths[key] = normalized
- if replaceable:
- replaced_receipt = True
- if receipt_warnings is not None:
- receipt_warnings.append(
- f"Replaced invalid saved coverage receipt {ref}: {failure}"
- )
- continue
- refs.append(ref)
- if repair_only and not replaced_receipt:
- continue
- try:
- _validate_schema_node(update, surface_schema, "coverage.surfaces")
- except ContractError:
- continue
- replaced_sources = {
- id(row)
- for _, row in matches
- if reopening
- or row.get("disposition") in ("needs_follow_up", surface.get("disposition"))
- }
- replaced.update(replaced_sources)
- existing = next(
- (entry for entry in updates if entry[0] == owner and entry[1] == update), None
- )
- if existing is None:
- receipt_paths.update(replacement_paths)
- updates.append((owner, update, replaced_sources))
- else:
- existing[2].update(replaced_sources)
- retained = next(
- entry[1] for entry in updates if entry[0] == owner and entry[1] == update
- )
- _inherit_coverage_sources(
- retained,
- [latest_surface, *(row for _, row in matches if id(row) in replaced_sources)],
- coverage_sources,
- )
- if id(latest_surface) in canonical_rows:
- canonical_rows.add(id(retained))
- return replaced, updates
+ return discarded_findings
+
+
+def _recover_preserved_worker_coverage(
+ scan_dir, relative, worker_id, draft, discarded_findings, warnings
+):
+ discarded_coverage = False
+ for field in ("surfaces", "explicitExclusions", "deferred"):
+ rows = draft["coverage"].get(field)
+ for row in rows if isinstance(rows, list) else []:
+ if isinstance(row, dict):
+ row.setdefault("id", _saved_coverage_id(row))
+ if field == "surfaces":
+ row.setdefault("receiptRefs", [])
+ source_warnings = []
+ discarded_fields = _recover_unsealed_coverage(
+ draft["coverage"],
+ Path(__file__).resolve().parents[2] / "schemas",
+ scan_dir,
+ source_warnings,
+ discarded_findings,
+ # Reconcile local IDs and valid outcomes against worker history below.
+ worker_draft=True,
+ )
+ if discarded_fields & {
+ "surfaces",
+ "explicitExclusions",
+ "deferred",
+ "resolvedDeferred",
+ }:
+ discarded_coverage = True
+ for row in draft["coverage"]["deferred"]:
+ row["workerId"] = worker_id
+ warnings.extend(f"{relative}: {warning}" for warning in source_warnings)
+ return discarded_coverage
def merge_saved_results(
@@ -666,6 +452,7 @@ def merge_saved_results(
except (ContractError, OSError, ValueError) as exc:
raise ContractError("Frozen stopped-scan checkpoint set is incomplete.") from exc
parent: dict[str, Any] | None = None
+ frozen_parent_path: str | None = None
parent_manifest: dict[str, Any] | None = None
parent_is_canonical = False
parent_modified = 0
@@ -839,7 +626,9 @@ def merge_saved_results(
latest_reducer = None
discarded_by_source: dict[str, list[str]] = {}
- for relative, worker_id in paths.items():
+ pending = deque(paths.items())
+ while pending:
+ relative, worker_id = pending.popleft()
try:
draft, digest, observed = _read_saved_result(
scan_dir, relative, scan_id, kind="dedup" if relative in reducer_paths else None
@@ -855,97 +644,27 @@ def merge_saved_results(
saved_inputs[relative] = draft["inputCheckpoint"]
continue
if preserve_canonical_parent and worker_id is not None:
- # Recover imported records without relaxing canonical parent validation.
- if "complete" in draft and not isinstance(draft["complete"], bool):
- draft["complete"] = False
- warnings.append(
- f"{relative}: Invalid complete marker; retained as a progress checkpoint."
- )
- schema_dir = Path(__file__).resolve().parents[2] / "schemas"
- if "threatModel" in draft:
- try:
- model = draft["threatModel"]
- if isinstance(model, dict):
- model = {**model, "origin": "recovered"}
- schema = _read_json(schema_dir / "scan-manifest.schema.json")
- _validate_schema_node(
- model,
- schema["properties"]["scan"]["properties"]["threatModel"],
- "threatModel",
- schema,
- )
- _require_safe_json_value(model, "threatModel")
- except ContractError as exc:
- draft.pop("threatModel")
- warnings.append(f"{relative}: Skipped malformed threat model: {exc}.")
- directory = Path(relative).parent
- if directory.name == "checkpoints":
- directory = directory.parent
- recovered_findings = []
- for index, finding in enumerate(draft["findings"]):
- needs_identity = isinstance(finding, dict) and "identity" not in finding
- _ensure_finding_identity(finding)
- recovered = {"scanId": scan_id, "findings": [finding]}
- source_warnings: list[str] = []
- discarded = _recover_unsealed_findings(
- {"scan": {"id": scan_id, "target": binding["target"]}},
- recovered,
- schema_dir,
- scan_dir / directory,
- source_warnings,
- )
- if recovered["findings"]:
- try:
- _validate_current_findings(recovered, schema_dir)
- except ContractError as exc:
- warning = f"Skipped malformed finding: {exc}."
- source_warnings.append(warning)
- discarded.append(warning)
- recovered["findings"] = []
- if discarded:
- discarded_by_source.setdefault(relative, []).extend(discarded)
- for recovered_finding in recovered["findings"]:
- if needs_identity:
- # Reconcile anonymous revisions before assigning sibling identities.
- recovered_finding.pop("identity", None)
- writeup = recovered_finding.pop("writeup", None)
- if writeup is not None:
- source_warnings.append(
- "Retained worker writeup at "
- f"{directory.as_posix()}/{writeup['reportPath']}; "
- "the parent report renders this finding inline."
- )
- recovered_findings.extend(recovered["findings"])
- warnings.extend(
- f"{relative} finding {index + 1}: {warning}" for warning in source_warnings
- )
- draft["findings"] = recovered_findings
- for field in ("surfaces", "explicitExclusions", "deferred", "openQuestions"):
- rows = draft["coverage"].get(field)
- if not isinstance(rows, list):
- continue
- for index, row in enumerate(rows):
- if field == "openQuestions" and isinstance(row, str):
- row = rows[index] = {"question": row}
- if not isinstance(row, dict):
- continue
- row["workerId"] = worker_id
- refs = row.get("receiptRefs")
- if not isinstance(refs, list):
- continue
- for ref_index, ref in enumerate(refs):
- if not isinstance(ref, str):
- continue
- try:
- normalized = _require_portable_relative_path(ref, "receiptRef")
- except ContractError:
- # Keep malformed references for the finalizer's diagnostics.
- continue
- if normalized.startswith("artifacts/"):
- refs[ref_index] = f"{directory.as_posix()}/{normalized}"
+ discarded = _recover_preserved_worker_source(
+ scan_dir,
+ relative,
+ scan_id,
+ binding,
+ worker_id,
+ next(worker for worker in workers if worker["id"] == worker_id),
+ draft,
+ warnings,
+ )
+ if discarded:
+ discarded_by_source[relative] = discarded
# Recovery expects coverage, but reducer results only contain findings
# and context. Add an empty value after hashing the original result.
sources.append((relative, {"coverage": {}, **draft}, worker_id))
+ if frozen_source_digests is None:
+ pending.extend(
+ receipt_checkpoints(
+ relative, worker_id, paths, partial(_checkpoint_paths, scan_dir)
+ ).items()
+ )
except (ContractError, OSError, ValueError) as exc:
if (scan_dir / relative).exists():
warnings.append(f"Preserved unreadable checkpoint {relative}: {exc}")
@@ -953,7 +672,7 @@ def merge_saved_results(
if frozen_source_digests.keys() - source_digests.keys():
raise ContractError("Frozen stopped-scan checkpoint set is incomplete.")
- # Frozen observations retain checkpoint selection even if a worker moves its head.
+ # Frozen observations keep checkpoint selection after a worker moves its head.
headed_workers = {
worker_attempts[_checkpoint_head_directory(head).as_posix()][0]
for head in saved_heads
@@ -984,6 +703,11 @@ def merge_saved_results(
selected_observations[selected] = max(selected_observations.get(selected, order), order)
source_order.update(selected_observations)
+ # Keep frozen source digests over the original data, before discarding markers
+ # that never passed through the worker draft writer.
+ retain_recorded_receipt_ownership(
+ sources, _digest, binding["scope"]["includePaths"], finding_scope
+ )
drafts_by_path = {relative: draft for relative, draft, _ in sources}
if frozen_source_digests is not None and "canonical" in parent_context:
canonical = parent_context["canonical"]
@@ -995,6 +719,172 @@ def merge_saved_results(
if "head" in parent_context:
head = parent_context["head"]
parent_heads = [(source_order[head][1], f"checkpoints/{saved_heads[head]}")]
+
+ if parent is None and frozen_source_digests is not None and "canonical" not in parent_context:
+ # Older host snapshots remain evidence; only the newest is the current parent.
+ saved_parents = {
+ relative: draft
+ for relative, draft, _ in sources
+ if relative.startswith("checkpoints/")
+ and isinstance(draft.get("previousParentCheckpoints"), list)
+ }
+ previous_parents = {
+ relative
+ for draft in saved_parents.values()
+ for relative in draft["previousParentCheckpoints"]
+ if isinstance(relative, str)
+ }
+ for relative, draft in saved_parents.items():
+ if relative not in previous_parents:
+ frozen_parent_path, parent = relative, draft
+ parent_is_canonical = draft.get("canonicalParent") is True
+ break
+ projected_coverage = parent["coverage"] if parent else {}
+ # V1 persists the review projection in the parent, without reducer sourceCoverage.
+
+ coverage_receipts = partial(resolve_coverage_receipts, scan_dir)
+
+ def canonical_coverage_record(field: str, item: dict[str, Any]) -> dict[str, Any]:
+ refs = item.get("receiptRefs", [])
+ if field != "surfaces" or not isinstance(refs, list):
+ return item
+ origin = projection_origins.get(id(item))
+ if origin is not None and (worker := workers_by_id.get(origin[0])) is not None:
+ refs = coverage_receipts(item, worker, origin[1])
+ result = {
+ **item,
+ "receiptRefs": [Path(ref).as_posix() if isinstance(ref, str) else ref for ref in refs],
+ }
+ if isinstance(provenance := result.get("provenance"), dict):
+ result["provenance"] = {
+ key: value for key, value in provenance.items() if key != "scanReceiptRefs"
+ }
+ return result
+
+ def retained_coverage_record(field: str, item: dict[str, Any]) -> dict[str, Any] | None:
+ return projection.retained_record(coverage, field, item, canonical_coverage_record)
+
+ reconciled_surface_ids: dict[tuple[str | None, str], str] = {}
+
+ def project_missing_record(
+ field: str,
+ item: dict[str, Any],
+ index: int,
+ worker: Any,
+ source: dict[str, Any],
+ *,
+ receipts_resolved: bool = False,
+ ) -> dict[str, Any]:
+ # Retain projectDiscoveryCoverage worker ownership.
+ prefix = f"{worker['id']}-attempt-{worker['attempt']}"
+
+ def accepted_positions(surface: dict[str, Any]) -> list[int]:
+ return [
+ position
+ for position, accepted in accepted_surfaces.get((worker["id"], surface["id"]), [])
+ if canonical_coverage_record("surfaces", accepted)
+ == canonical_coverage_record("surfaces", surface)
+ ]
+
+ def surface_id(surface: dict[str, Any], offset: int) -> str:
+ source_id = surface.get("id")
+ if id(source) not in accepted_coverage_sources and isinstance(source_id, str):
+ positions = accepted_positions(surface)
+ if len(positions) == 1:
+ offset = positions[0]
+ elif not positions:
+ return f"{prefix}-surface-{offset}-{_digest(surface)[:16]}"
+ return f"{prefix}-surface-{offset}"
+
+ result = copy.deepcopy(item)
+ provenance = result.get("provenance")
+ if not isinstance(provenance, dict):
+ provenance = {}
+ for key in ("workerId", "attempt", "sourceId", "candidateId"):
+ provenance.pop(key, None)
+ provenance.update(workerId=worker["id"], attempt=worker["attempt"])
+ result["provenance"] = provenance
+ source_rows = source.get(field, [])
+ inferred_id = (
+ isinstance(source_rows, list)
+ and index <= len(source_rows)
+ and id(source_rows[index - 1]) in inferred_coverage_ids
+ )
+ for key, name in (("id", "sourceId"), ("candidateId", "candidateId")):
+ if key == "id" and inferred_id:
+ continue
+ if key in item:
+ result["provenance"][name] = item[key]
+ if isinstance(item.get("candidateId"), str):
+ candidate_digest = hashlib.sha256(javascript_utf8(item["candidateId"])).hexdigest()
+ result["candidateId"] = f"{prefix}-candidate-{candidate_digest}"
+ if field == "surfaces":
+ original = source_rows[index - 1] if isinstance(source_rows, list) else item
+ result["id"] = surface_id(original, index)
+ if not receipts_resolved and (origin := projection_origins.get(id(original))):
+ result["receiptRefs"] = coverage_receipts(result, worker, origin[1])
+ previous = retained_coverage_record("surfaces", result)
+ if previous is not None and isinstance(previous.get("id"), str):
+ result["id"] = previous["id"]
+ else:
+ surfaces = coverage.get("surfaces", [])
+ if any(
+ isinstance(row, dict) and row.get("id") == result["id"]
+ for row in (surfaces if isinstance(surfaces, list) else [])
+ ):
+ result["id"] = f"{result['id']}-{_digest(result)[:16]}"
+ elif field == "deferred":
+ result["id"] = f"{prefix}-deferred-{index}"
+ if isinstance(item.get("surfaceIds"), list):
+ surface_ids: dict[str, str] = {}
+ surfaces = projected_coverage.get("surfaces", [])
+ for surface in surfaces if isinstance(surfaces, list) else []:
+ if not isinstance(surface, dict) or not isinstance(surface.get("id"), str):
+ continue
+ provenance = surface.get("provenance", {})
+ if (
+ isinstance(provenance, dict)
+ and provenance.get("workerId") == worker["id"]
+ and provenance.get("attempt") == worker["attempt"]
+ and isinstance(provenance.get("sourceId"), str)
+ ):
+ surface_ids.setdefault(provenance["sourceId"], surface["id"])
+ surfaces = source.get("surfaces", [])
+ source_surface_ids: dict[str, tuple[int, dict[str, Any]]] = {}
+ for offset, surface in enumerate(surfaces if isinstance(surfaces, list) else [], 1):
+ if isinstance(surface, dict) and isinstance(surface.get("id"), str):
+ source_surface_ids.setdefault(surface["id"], (offset, surface))
+ for (owner, source_id), accepted in accepted_surfaces.items():
+ if owner == worker["id"] and accepted:
+ source_surface_ids.setdefault(source_id, accepted[0])
+ for source_id, (offset, surface) in source_surface_ids.items():
+ if source_id not in item["surfaceIds"]:
+ continue
+ if (worker["id"], source_id) in reconciled_surface_ids:
+ surface_ids[source_id] = reconciled_surface_ids[(worker["id"], source_id)]
+ continue
+ projected_id = surface_id(surface, offset)
+ if id(source) in accepted_coverage_sources or accepted_positions(surface):
+ origin = projection_origins.get(id(surface))
+ projected = project_missing_record(
+ "surfaces", surface, offset, worker, origin[4] if origin else source
+ )
+ if origin := projection_origins.get(id(surface)):
+ projected["receiptRefs"] = coverage_receipts(
+ projected, worker, origin[1]
+ )
+ projected_id = projected["id"]
+ previous = retained_coverage_record("surfaces", projected)
+ if previous is not None and isinstance(previous.get("id"), str):
+ projected_id = previous["id"]
+ surface_ids[source_id] = projected_id
+ result["surfaceIds"] = [
+ surface_ids.get(value, value) if isinstance(value, str) else value
+ for value in item["surfaceIds"]
+ ]
+ return result
+
+ workers_by_id = {worker["id"]: worker for worker in workers}
latest_reducer_key = (
(reducer["completed_at"] or "", reducer["id"], int(reducer["attempt"] or 0))
if reducer is not None and latest_reducer in drafts_by_path
@@ -1023,89 +913,158 @@ def merge_saved_results(
modified = source_order[parent_path][1]
if parent is None or modified > parent_modified:
parent = draft
+ frozen_parent_path = (
+ parent_path
+ if isinstance(draft.get("previousParentCheckpoints"), list)
+ else None
+ )
parent_modified = modified
- parent_is_canonical = False
- elif modified == parent_modified and draft != parent:
+ parent_is_canonical = draft.get("canonicalParent") is True
+ elif modified == parent_modified and _parent_observation_content(
+ draft
+ ) != _parent_observation_content(parent):
parent = _merge_tied_parent_observations(parent, draft)
parent_is_canonical = False
+ # A reducer supplies findings but cannot supersede saved parent coverage.
+ saved_parent = parent
parent_is_root = parent is not None
if parent is None and latest_reducer is not None:
parent = drafts_by_path[latest_reducer]
- if preserve_canonical_parent:
- project_worker_coverage, surface_links = False, []
- else:
- project_worker_coverage, surface_links = _prepare_worker_coverage_recovery(
- sources, parent, source_order, enabled=project_worker_coverage
- )
- # Keep the actual retained row even when recovery adds previous findings.
- linked_surface_sources = {id(surface) for _, _, surface, _, _ in surface_links}
- linked_surface_outputs: dict[int, dict[str, Any]] = {}
- worker_row_owners: dict[int, str] = {}
- worker_surface_outputs: dict[tuple[str, str], dict[int, dict[str, Any]]] = {}
-
- def record_coverage_owner(retained: Any, item: Any, owner: str | None, field: str) -> None:
- # Parent aggregates can carry worker metadata without originating in
- # that worker's retained source. Track the actual source of output rows.
- if project_worker_coverage and owner is not None and isinstance(retained, dict):
- worker_row_owners[id(retained)] = owner
- if field == "surfaces" and isinstance(item.get("id"), str):
- worker_surface_outputs.setdefault((owner, item["id"]), {})[id(retained)] = retained
-
- def append_coverage_row(output: list[Any], item: Any, owner: str | None, field: str) -> Any:
- retained = _project_worker_coverage_row(
- item, owner if project_worker_coverage else None, field
- )
- if retained not in output:
- output.append(retained)
- retained = output[output.index(retained)]
- record_coverage_owner(retained, item, owner, field)
- _inherit_coverage_sources(retained, [item], coverage_sources)
- if preserve_canonical_parent:
- if id(item) in canonical_rows:
- canonical_rows.add(id(retained))
- if field == "surfaces":
- retained_surface_rows.add(id(item))
+ projected_coverage = saved_parent["coverage"] if saved_parent else {}
+ reviews = projected_coverage.get("reviews")
+ reviewed_attempts = {
+ (review.get("workerId"), review.get("attempt"))
+ for review in (reviews if isinstance(reviews, list) else [])
+ if isinstance(review, dict)
+ and isinstance(review.get("workerId"), str)
+ and isinstance(review.get("attempt"), int)
+ }
+ # Reconcile raw source IDs and timestamps before projecting accepted outputs.
+ accepted_sources = {
+ relative
+ for relative, draft, worker_id in sources
+ if (worker := workers_by_id.get(worker_id)) is not None
+ and worker["status"] == "succeeded"
+ and worker["result_manifest_path"] == str(scan_dir / relative)
+ and draft.get("complete") is not False
+ }
+ reviewed_attempts.update(
+ (worker_id, workers_by_id[worker_id]["attempt"])
+ for relative, _, worker_id in sources
+ if relative in accepted_sources
+ )
+ projection = prepare_saved_coverage_projection(
+ sources,
+ parent,
+ source_order,
+ project_worker_coverage=project_worker_coverage,
+ preserve_canonical_parent=preserve_canonical_parent,
+ receipt_resolver=lambda relative, row, owner: resolve_coverage_receipts(
+ scan_dir, row, workers_by_id[owner], relative
+ ),
+ )
+ project_worker_coverage = projection.project_worker_coverage
+
+ def retain_coverage_row(output, item, owner, field, original=None, qualified=False):
+ retained = projection.retain_coverage_row(output, item, owner, field, original, qualified)
+ source = item if original is None else original
+ _inherit_coverage_sources(retained, [item, source], coverage_sources)
+ if preserve_canonical_parent and (
+ id(item) in canonical_rows or id(source) in canonical_rows
+ ):
+ canonical_rows.add(id(retained))
+ if field == "surfaces":
+ retained_surface_rows.add(id(source))
return retained
- def linked_tasks(
- task: dict[str, Any],
- published_id: str,
- source_task: dict[str, Any] | None,
- candidates: list[tuple[str | None, Any]],
- ) -> list[Any]:
- expected = (
- _project_worker_coverage_row(source_task, source_task["workerId"], "deferred")
- if source_task is not None
- else None
+ accepted_coverage_sources = {
+ id(draft["coverage"]) for relative, draft, _ in sources if relative in accepted_sources
+ }
+ accepted_surfaces: dict[tuple[str, str], list[tuple[int, dict[str, Any]]]] = {}
+ for relative, draft, worker_id in sources:
+ if relative not in accepted_sources:
+ continue
+ surfaces = draft["coverage"].get("surfaces", [])
+ for index, surface in enumerate(surfaces if isinstance(surfaces, list) else [], 1):
+ if isinstance(surface, dict) and isinstance(surface.get("id"), str):
+ accepted_surfaces.setdefault((worker_id, surface["id"]), []).append(
+ (index, surface)
+ )
+
+ for _, draft, _ in sources:
+ questions = draft["coverage"].get("openQuestions")
+ if isinstance(questions, list):
+ draft["coverage"]["openQuestions"] = [
+ {"question": item.strip()} if isinstance(item, str) else item for item in questions
+ ]
+ projection_origins = {
+ id(item): (worker_id, relative, field, index, draft["coverage"])
+ for relative, draft, worker_id in sources
+ for field in ("surfaces", "explicitExclusions", "deferred", "openQuestions")
+ for index, item in enumerate(
+ draft["coverage"].get(field, [])
+ if isinstance(draft["coverage"].get(field, []), list)
+ else [],
+ 1,
)
- return [
- retained
- for owner, retained in candidates
- if retained == task
- or (
- expected is not None
- and owner == expected["workerId"]
- and isinstance(retained, dict)
- and retained.get("id") == expected.get("id")
- and isinstance(retained.get("surfaceIds"), list)
- and published_id in retained["surfaceIds"]
- )
- ]
+ if isinstance(item, dict)
+ }
- def linked_surface(surface, outputs, named_outputs):
- retained = outputs.get(id(surface))
- if (
- retained is None
- and isinstance(owner := surface.get("workerId"), str)
- and isinstance(identity := surface.get("id"), str)
- ):
- matches = named_outputs.get((owner, identity), {})
- if len(matches) == 1:
- retained = next(iter(matches.values()))
- return retained
+ def project_record(
+ field: str,
+ item: dict[str, Any],
+ source_item: dict[str, Any] | None = None,
+ *,
+ receipts_resolved: bool = False,
+ ) -> dict[str, Any]:
+ original = item if source_item is None else source_item
+ origin = projection_origins.get(id(original))
+ if origin is None:
+ return item
+ worker_id, relative, _, index, source = origin
+ if relative not in accepted_sources:
+ directory = Path(relative).parent
+ if directory.name == "checkpoints":
+ directory = directory.parent
+ worker = workers_by_id.get(worker_id)
+ if worker is None or worker_attempts.get(directory.as_posix(), (None, None)) != (
+ worker_id,
+ worker["attempt"],
+ ):
+ return item
+ # Reproduced outputs and links from omitted checkpoints use the
+ # accepted worker's surface namespace.
+ accepted = next(
+ (
+ (path, draft["coverage"])
+ for path, draft, owner in sources
+ if owner == worker_id and path in accepted_sources
+ ),
+ None,
+ )
+ if accepted is not None and source == accepted[1]:
+ relative, source = accepted
+ elif relative not in selected_observations or accepted is None:
+ if field == "deferred" and accepted is not None and "surfaceIds" in item:
+ return {
+ **item,
+ "surfaceIds": project_missing_record(
+ field, item, index, worker, accepted[1]
+ )["surfaceIds"],
+ }
+ return item
+ worker = workers_by_id[worker_id]
+ result = project_missing_record(
+ field, item, index, worker, source, receipts_resolved=receipts_resolved
+ )
+ if field == "surfaces" and not receipts_resolved:
+ result["receiptRefs"] = coverage_receipts(result, worker, relative)
+ return result
all_sources = ([("parent", parent, None)] if parent else []) + sources
+ source_owners = {relative: owner for relative, _, owner in all_sources}
+ inferred_coverage_ids: set[int] = set()
# Older checkpoints can omit IDs already assigned in their published output.
for field in ("deferred", "surfaces"):
named_rows: dict[str | None, list[dict[str, Any]]] = {}
@@ -1142,6 +1101,7 @@ def merge_saved_results(
None,
)
if identity is not None:
+ inferred_coverage_ids.add(id(row))
row["id"] = identity
reserved.add(identity)
@@ -1161,33 +1121,15 @@ def merge_saved_results(
for relative, draft, worker_id in sources:
if worker_id is None:
continue
- for field in ("surfaces", "explicitExclusions", "deferred"):
- rows = draft["coverage"].get(field)
- for row in rows if isinstance(rows, list) else []:
- if isinstance(row, dict):
- row.setdefault("id", _saved_coverage_id(row))
- if field == "surfaces":
- row.setdefault("receiptRefs", [])
- source_warnings = []
- discarded_fields = _recover_unsealed_coverage(
- draft["coverage"],
- Path(__file__).resolve().parents[2] / "schemas",
+ if _recover_preserved_worker_coverage(
scan_dir,
- source_warnings,
+ relative,
+ worker_id,
+ draft,
discarded_by_source.get(relative, []),
- # Reconcile local IDs and valid outcomes against worker history below.
- worker_draft=True,
- )
- if discarded_fields & {
- "surfaces",
- "explicitExclusions",
- "deferred",
- "resolvedDeferred",
- }:
+ warnings,
+ ):
discarded_coverage_sources.add(relative)
- for row in draft["coverage"]["deferred"]:
- row["workerId"] = worker_id
- warnings.extend(f"{relative}: {warning}" for warning in source_warnings)
if parent is None and not sources:
return None
@@ -1456,6 +1398,119 @@ def merge_saved_results(
legacy_identities[key] = None
source_order["parent"] = (0, parent_modified)
+
+ def projection_key(field: str, item: dict[str, Any]) -> tuple[str, str]:
+ # Publication may rename canonical IDs and add retained finding history.
+ ignored = {"id", "previousFindings"} if field == "surfaces" else {"id"}
+ content = {
+ key: value
+ for key, value in canonical_coverage_record(field, item).items()
+ if key not in ignored
+ }
+ return field, _digest(content)
+
+ projected_sources: dict[tuple[str, str], tuple[str, dict[str, Any], tuple[int, int]]] = {}
+ accepted_pending_candidates: set[tuple[str, int, str]] = set()
+ selected_reported_candidates: dict[tuple[str, str], tuple[int, int]] = {}
+ for relative, draft, worker_id in sources:
+ if relative not in selected_observations:
+ continue
+ for finding in draft["findings"]:
+ if isinstance(finding, dict) and valid_finding(finding):
+ candidate_id = finding_candidate_id(finding)
+ if candidate_id:
+ key = (worker_id, candidate_id)
+ selected_reported_candidates[key] = max(
+ selected_reported_candidates.get(key, source_order[relative]),
+ source_order[relative],
+ )
+ for relative, draft, worker_id in sources:
+ if relative not in accepted_sources:
+ continue
+ worker = workers_by_id[worker_id]
+ reported = {
+ finding_candidate_id(finding)
+ for finding in draft["findings"]
+ if isinstance(finding, dict) and valid_finding(finding)
+ }
+ for field in ("surfaces", "deferred", "explicitExclusions"):
+ rows = draft["coverage"].get(field, [])
+ for index, row in enumerate(rows if isinstance(rows, list) else [], 1):
+ if not isinstance(row, dict):
+ continue
+ if (
+ field == "deferred"
+ and isinstance(candidate_id := row.get("candidateId"), str)
+ and candidate_id not in reported
+ and selected_reported_candidates.get((worker_id, candidate_id), (-1, -1))
+ < source_order[relative]
+ ):
+ accepted_pending_candidates.add((worker_id, worker["attempt"], candidate_id))
+ projected = project_missing_record(field, row, index, worker, draft["coverage"])
+ if field == "surfaces":
+ projected["receiptRefs"] = coverage_receipts(projected, worker, relative)
+ key = projection_key(field, projected)
+ projected_sources[key] = (worker_id, row, source_order[relative])
+
+ def record_source(
+ field: str, owner: str | None, item: dict[str, Any]
+ ) -> tuple[str | None, dict[str, Any]]:
+ # Parent-authored provenance is data, not authority over another worker.
+ if (source := projected_sources.get(projection_key(field, item))) and (
+ owner is None or owner == source[0]
+ ):
+ return source[0], source[1]
+ return owner, item
+
+ def record_key(field: str, owner: str | None, item: dict[str, Any]) -> tuple[str | None, Any]:
+ source_owner, original = record_source(field, owner, item)
+ identity = original.get("id")
+ return source_owner, identity if isinstance(identity, str) else None
+
+ def coverage_candidate(
+ owner: str | None, item: dict[str, Any]
+ ) -> tuple[str | None, str] | None:
+ for field in ("surfaces", "deferred", "explicitExclusions"):
+ source_owner, original = record_source(field, owner, item)
+ if original is not item:
+ owner, item = source_owner, original
+ break
+ else:
+ # A later parent can complete a known worker proof gap, but its
+ # claimed provenance cannot overturn an accepted worker finding.
+ provenance = item.get("provenance")
+ if owner is None and isinstance(provenance, dict):
+ source_owner = provenance.get("workerId")
+ attempt = provenance.get("attempt")
+ candidate_id = provenance.get("candidateId")
+ if (
+ isinstance(source_owner, str)
+ and isinstance(attempt, int)
+ and isinstance(candidate_id, str)
+ and (source_owner, attempt, candidate_id) in accepted_pending_candidates
+ ):
+ return source_owner, candidate_id
+ candidate_id = (
+ _deferred_candidate_id(item, owner, ambiguous_deferred)
+ if any(key in item for key in ("candidate", "finding"))
+ else item.get("candidateId")
+ )
+ return (owner, candidate_id) if isinstance(candidate_id, str) else None
+
+ def candidate_order(
+ relative: str, owner: str | None, item: dict[str, Any] | None = None
+ ) -> tuple[int, int]:
+ order = source_order[relative]
+ if source_owners[relative] is None and owner is not None:
+ if item is not None:
+ for field in ("surfaces", "deferred", "explicitExclusions"):
+ key = projection_key(field, item)
+ if (source := projected_sources.get(key)) and source[0] == owner:
+ return source[2]
+ attempt = int(workers_by_id[owner]["attempt"] or 0) if owner in headed_workers else 0
+ return attempt, order[1]
+ return order
+
deferred_rows = {
relative: _coverage_rows(draft["coverage"], "deferred")
for relative, draft, _ in all_sources
@@ -1478,12 +1533,13 @@ def merge_saved_results(
# Later rewrites cannot make an ID-only closure identify one of those tasks.
ambiguous_deferred: set[tuple[str | None, str]] = set()
for relative, _, owner in all_sources:
- by_id: dict[str, dict[str, Any]] = {}
+ by_id: dict[tuple[str | None, Any], dict[str, Any]] = {}
candidate_aliases = {
- identity
+ (source_owner, identity)
for row in deferred_rows[relative]
if isinstance(row, dict) and _has_candidate_metadata(row)
- for identity in (row.get("id"), row.get("candidateId"))
+ for source_owner, original in [record_source("deferred", owner, row)]
+ for identity in (original.get("id"), original.get("candidateId"))
if isinstance(identity, str)
}
for row in deferred_rows[relative]:
@@ -1491,12 +1547,27 @@ def merge_saved_results(
continue
if _has_candidate_metadata(row):
continue
- if identity in candidate_aliases or (identity in by_id and row != by_id[identity]):
- ambiguous_deferred.add((owner, identity))
- by_id[identity] = row
+ key = record_key("deferred", owner, row)
+ if key in candidate_aliases or (key in by_id and row != by_id[key]):
+ ambiguous_deferred.add(key)
+ by_id[key] = row
current_drafts = ([("parent", parent, None)] if parent else []) + [
source for source in sources if source[0] in current_results | selected_observations.keys()
]
+
+ pending_surface_orders: dict[tuple[str | None, str], tuple[int, int]] = {}
+ for relative, draft, owner in all_sources:
+ surfaces = draft["coverage"].get("surfaces", [])
+ for surface in surfaces if isinstance(surfaces, list) else []:
+ if (
+ isinstance(surface, dict)
+ and surface.get("disposition") == "needs_follow_up"
+ and (candidate := coverage_candidate(owner, surface)) is not None
+ ):
+ order = candidate_order(relative, candidate[0], surface)
+ pending_surface_orders[candidate] = max(
+ pending_surface_orders.get(candidate, order), order
+ )
# Generic closures belong to one logical scan or worker, just like candidates.
# Keep them when recovering a terminal checkpoint without its canonical write.
closed_deferred: dict[tuple[str | None, str], tuple[tuple[int, int], dict[str, Any], str]] = {}
@@ -1507,23 +1578,29 @@ def merge_saved_results(
coverage_schema = _read_json(
Path(__file__).resolve().parents[2] / "schemas" / "coverage.schema.json"
)["properties"]
+
+ projected_closure_sources = _projected_worker_closure_sources(
+ sources, source_order, worker_attempts, coverage_schema["resolvedDeferred"]
+ )
+
for relative, draft, owner in all_sources:
- order = source_order[relative]
for item in deferred_rows[relative]:
if not isinstance(item, dict):
continue
+ source_owner, original = record_source("deferred", owner, item)
+ order = candidate_order(relative, source_owner, item)
if isinstance(item.get("candidateId"), str) or "candidate" in item or "finding" in item:
candidate_ids.update(
- (owner, identity)
- for identity in (item.get("id"), item.get("candidateId"))
+ (source_owner, identity)
+ for identity in (original.get("id"), original.get("candidateId"))
if isinstance(identity, str)
)
- identity = item.get("id") or item.get("candidateId")
+ identity = original.get("id") or original.get("candidateId")
if isinstance(identity, str):
- key = (owner, identity)
+ key = (source_owner, identity)
previous = active_deferred.get(key)
if previous is None or order > previous[0]:
- active_deferred[key] = (order, item, relative)
+ active_deferred[key] = (order, original, relative)
if (
relative == "parent"
or relative in current_results
@@ -1540,11 +1617,19 @@ def merge_saved_results(
accepted_deferred_orders.get(key, order), order
)
for closure in _resolved_deferred_rows(draft, coverage_schema["resolvedDeferred"]):
- key = (owner, closure["id"])
+ closure_owner, closure_relative = owner, relative
+ if owner is None and (
+ source := projected_closure_sources.get((closure["id"], closure["reason"]))
+ ):
+ # A copied parent closeout keeps its worker's identity and observation
+ # time, so a later parent write cannot hide reopened worker work.
+ closure_owner, closure, closure_relative = source
+ key = (closure_owner, closure["id"])
+ order = candidate_order(closure_relative, closure_owner)
previous = closed_deferred.get(key)
if previous is None or order > previous[0]:
# The parent comes first, preserving its reason on equal timestamps.
- closed_deferred[key] = (order, closure, relative)
+ closed_deferred[key] = (order, closure, closure_relative)
reopened_rows: list[tuple[str | None, dict[str, Any]]] = []
reopened_generic: set[tuple[str | None, str]] = set()
for key, (order, _, _) in list(closed_deferred.items()):
@@ -1569,17 +1654,20 @@ def merge_saved_results(
if (
isinstance(row, dict)
and isinstance(identity := row.get("id"), str)
- and (owner, identity) in ambiguous_deferred
+ and record_key("deferred", owner, row) in ambiguous_deferred
and not _has_candidate_metadata(row)
and (owner, row) not in reopened_rows
):
reopened_rows.append((owner, row))
- parent_closures = [
- closure for (owner, _), (_, closure, _) in closed_deferred.items() if owner is None
+ retained_closures = [
+ closure
+ if owner is None
+ else _project_worker_closure(relative, owner, closure, worker_attempts, source_order)
+ for (owner, _), (_, closure, relative) in closed_deferred.items()
]
coverage.pop("resolvedDeferred", None)
- if parent_closures:
- coverage["resolvedDeferred"] = copy.deepcopy(parent_closures)
+ if retained_closures:
+ coverage["resolvedDeferred"] = copy.deepcopy(retained_closures)
if parent:
coverage["deferred"] = [
row
@@ -1598,41 +1686,84 @@ def merge_saved_results(
}
ordered_outcomes: dict[tuple[str | None, str], tuple[tuple[int, int], str]] = {}
- outcomes: list[tuple[str, str | None, str, str]] = []
+ pending_candidate_orders: dict[tuple[str | None, str], tuple[int, int]] = {}
+ outcomes: list[tuple[str, str | None, str, str, tuple[int, int]]] = []
for relative, draft, owner in current_drafts:
+ reported_candidates = {
+ finding_candidate_id(finding)
+ for finding in draft["findings"]
+ if isinstance(finding, dict) and valid_finding(finding)
+ }
+ pending_candidates = set()
+ for field in ("deferred", "surfaces"):
+ items = draft["coverage"].get(field, [])
+ for item in items if isinstance(items, list) else []:
+ if isinstance(item, dict) and (
+ field == "deferred" or item.get("disposition") == "needs_follow_up"
+ ):
+ if (pending := coverage_candidate(owner, item)) is not None:
+ if (
+ owner is not None
+ and pending[0] == owner
+ and pending[1] in reported_candidates
+ ):
+ continue
+ pending_candidates.add(pending)
+ order = candidate_order(relative, pending[0], item)
+ pending_candidate_orders[pending] = max(
+ pending_candidate_orders.get(pending, order), order
+ )
for finding in draft["findings"]:
if (
isinstance(finding, dict)
and valid_finding(finding)
and (candidate_id := finding_candidate_id(finding))
):
- outcomes.append((relative, owner, candidate_id, "reported"))
+ outcomes.append(
+ (relative, owner, candidate_id, "reported", candidate_order(relative, owner))
+ )
for field in ("surfaces", "explicitExclusions"):
for item in _coverage_rows(draft["coverage"], field):
if (
isinstance(item, dict)
and isinstance(item.get("candidateId"), str)
- and item.get("disposition") in ("rejected", "not_applicable")
+ and item.get("disposition") in ("reported", "rejected", "not_applicable")
+ and (candidate := coverage_candidate(owner, item)) is not None
+ and (item["disposition"] != "reported" or candidate[1] in reported_candidates)
+ and (candidate not in pending_candidates or item["disposition"] == "reported")
):
- outcomes.append((relative, owner, item["candidateId"], item["disposition"]))
+ outcomes.append(
+ (
+ relative,
+ candidate[0],
+ candidate[1],
+ item["disposition"],
+ candidate_order(relative, candidate[0], item),
+ )
+ )
+ ordered_candidates.update(pending_candidate_orders)
ordered_candidates.update(
(owner, candidate_id)
- for relative, owner, candidate_id, _ in outcomes
+ for relative, owner, candidate_id, _, _ in outcomes
if owner is not None and relative in selected_observations
)
# Reopened work and selected checkpoint outcomes follow the saved source order.
- for relative, owner, candidate_id, disposition in outcomes:
+ for relative, owner, candidate_id, disposition, order in outcomes:
key = (owner, candidate_id)
+ if pending_surface_orders.get(key, (-1, -1)) >= order:
+ continue
if key not in ordered_candidates:
if relative == "parent" or relative in current_results:
resolved.setdefault(key, disposition)
continue
- order = source_order[relative]
- if any(
- saved_owner == owner
- and _deferred_candidate_id(row, owner, ambiguous_deferred) == candidate_id
- and modified >= order
- for (saved_owner, _), (modified, row, _) in active_deferred.items()
+ if pending_candidate_orders.get(key, (-1, -1)) >= order or any(
+ (
+ coverage_candidate(saved_owner, row)
+ or (saved_owner, _deferred_candidate_id(row, saved_owner, ambiguous_deferred))
+ )
+ == key
+ and saved_order >= order
+ for (saved_owner, _), (saved_order, row, _) in active_deferred.items()
):
continue
if key not in ordered_outcomes or order > ordered_outcomes[key][0]:
@@ -1680,6 +1811,25 @@ def merge_saved_results(
represented_candidate_history.setdefault(candidate_key, set()).add(
_digest(_finding_content(original["finding"]))
)
+ # Apply reconciled outcomes to coverage already copied from the parent too.
+ for field in ("surfaces", "deferred"):
+ rows = coverage.get(field)
+ if isinstance(rows, list):
+ coverage[field] = [
+ row
+ for row in rows
+ if not (
+ isinstance(row, dict)
+ and (
+ (field == "deferred" and record_key(field, None, row) in closed_deferred)
+ or (
+ coverage_candidate(None, row) in resolved
+ and (field == "deferred" or row.get("disposition") == "needs_follow_up")
+ )
+ )
+ )
+ ]
+
selected_terminal_orders: dict[str, tuple[int, int]] = {}
for relative, draft, worker_id in sources:
if (
@@ -2271,13 +2421,20 @@ def merge_saved_results(
}
superseded = (
worker_id is None
- and parent is not None
- and parent.get("complete") is not False
+ and saved_parent is not None
+ and saved_parent.get("complete") is not False
and relative != "parent"
and source_order[relative] <= (0, parent_modified)
and (not stopped_parent_seal or relative in parent_preserved_sources)
+ and (
+ # Explicit recovery can admit checkpoints written after this parent.
+ frozen_parent_path is None
+ or relative == frozen_parent_path
+ or relative in saved_parent.get("previousParentCheckpoints", [])
+ )
) or (
- relative not in current_results
+ worker_id is not None
+ and relative not in current_results
and worker_result_order is not None
and (
relative not in selected_observations
@@ -2360,12 +2517,27 @@ def merge_saved_results(
else item.get("candidateId"),
str,
)
- and (worker_id, identity) in resolved
+ and (coverage_candidate(worker_id, item) or (worker_id, identity)) in resolved
and (field == "deferred" or item.get("disposition") == "needs_follow_up")
):
return False
return True
+ def source_surface_receipts(relative: str, row: dict[str, Any]) -> Any:
+ worker = workers_by_id.get(source_owners.get(relative))
+ return coverage_receipts(row, worker, relative) if worker else row.get("receiptRefs", [])
+
+ replaced_deferred = {
+ id(row)
+ for relative, _, owner in all_sources
+ for row in deferred_rows[relative]
+ if isinstance(row, dict)
+ and isinstance(row.get("id"), str)
+ and (key := record_key("deferred", owner, row)) not in candidate_ids
+ and key not in ambiguous_deferred
+ and (updated := accepted_deferred_orders.get(key)) is not None
+ and updated > candidate_order(relative, key[0], row)
+ }
ambiguous_surface_ids = {
(owner, identity)
for owner, row in reopened_rows
@@ -2395,76 +2567,23 @@ def merge_saved_results(
and not _has_candidate_metadata(row)
)
- replaced_deferred = {
- id(row)
- for relative, _, owner in all_sources
- for row in deferred_rows[relative]
- if isinstance(row, dict)
- and isinstance(identity := row.get("id"), str)
- and (key := (owner, identity)) not in candidate_ids
- and key not in ambiguous_deferred
- and (updated := accepted_deferred_orders.get(key)) is not None
- and updated > source_order[relative]
- }
- fallback_surfaces = []
- if surface_links and isinstance(coverage.get("surfaces"), list):
- # Plan the surviving associations before historical receipts can claim
- # a spelling needed by a surface that must be restored for its task.
- prospective_tasks = []
- prospective_surfaces = {}
- prospective_named = {}
- for relative, draft, owner in all_sources:
- selection = source_coverage_selection(relative, owner)
- for row in deferred_rows[relative]:
- if (
- coverage_row_is_retained("deferred", row, owner, selection)
- and id(row) not in replaced_deferred
- and not (
- isinstance(row, dict)
- and isinstance(row.get("id"), str)
- and (owner, row["id"]) in closed_deferred
- )
- ):
- prospective_tasks.append(
- (owner, _project_worker_coverage_row(row, owner, "deferred"))
- )
- for row in _coverage_rows(draft["coverage"], "surfaces"):
- if owner is None or not isinstance(row, dict):
- continue
- emitted = coverage_row_is_retained("surfaces", row, owner, selection)
- if not emitted and not ambiguous_pending_surface(row, owner):
- continue
- projected = _project_worker_coverage_row(row, owner, "surfaces")
- if isinstance(row.get("id"), str):
- named = prospective_named.setdefault((owner, row["id"]), {})
- projected = next(
- (existing for existing in named.values() if existing == projected),
- projected,
- )
- named[id(projected)] = projected
- if emitted:
- prospective_surfaces[id(row)] = projected
- prospective_tasks.extend(
- (owner, _project_worker_coverage_row(row, owner, "deferred"))
- for owner, row in reopened_rows
- if (owner, _deferred_candidate_id(row, owner, ambiguous_deferred)) not in resolved
- )
- bound = {
- (id(retained), published_id)
- for task, published_id, surface, _, task_source in surface_links
- if linked_surface(surface, prospective_surfaces, prospective_named) is not None
- for retained in linked_tasks(task, published_id, task_source, prospective_tasks)
- }
- for task, published_id, _, parent_surface, task_source in surface_links:
- for retained in linked_tasks(task, published_id, task_source, prospective_tasks):
- key = (id(retained), published_id)
- if key not in bound:
- fallback_surfaces.append(parent_surface)
- bound.add(key)
+ fallback_surfaces = projection.fallback_surfaces(
+ coverage,
+ all_sources,
+ deferred_rows,
+ reopened_rows,
+ selection_for_source=source_coverage_selection,
+ row_is_retained=coverage_row_is_retained,
+ replaced_deferred=replaced_deferred,
+ closed_deferred=closed_deferred,
+ ambiguous_surface_ids=ambiguous_surface_ids,
+ deferred_is_resolved=lambda owner, row: (
+ (owner, _deferred_candidate_id(row, owner, ambiguous_deferred)) in resolved
+ ),
+ )
replaced_surfaces, surface_updates = _generic_surface_updates(
all_sources,
- source_order,
closed_deferred,
active_deferred,
resolved,
@@ -2472,8 +2591,11 @@ def merge_saved_results(
coverage_schema["surfaces"]["items"],
deferred_rows,
ambiguous_deferred,
- canonical_rows,
- coverage_sources,
+ record_source,
+ candidate_order,
+ source_surface_receipts,
+ canonical_rows=canonical_rows,
+ coverage_sources=coverage_sources,
receipt_root=scan_dir if project_worker_coverage else None,
receipt_warnings=warnings,
fallback_surfaces=fallback_surfaces,
@@ -2488,7 +2610,17 @@ def merge_saved_results(
)
replaced_rows = {
"surfaces": replaced_surfaces,
- "deferred": replaced_deferred,
+ "deferred": {
+ id(row)
+ for relative, _, owner in all_sources
+ for row in deferred_rows[relative]
+ if isinstance(row, dict)
+ and isinstance(identity := row.get("id"), str)
+ and (key := record_key("deferred", owner, row)) not in candidate_ids
+ and key not in ambiguous_deferred
+ and (updated := accepted_deferred_orders.get(key)) is not None
+ and updated > candidate_order(relative, key[0], row)
+ },
}
for field, replaced in replaced_rows.items():
if parent and isinstance(coverage.get(field), list):
@@ -2497,10 +2629,28 @@ def merge_saved_results(
removed_parent = [row for row in previous if id(row) in replaced]
coverage[field] = [row for row in coverage[field] if row not in removed_parent]
if isinstance(coverage.get("surfaces"), list):
- for owner, surface, replaced_sources in surface_updates:
- retained = append_coverage_row(coverage["surfaces"], surface, owner, "surfaces")
- for source_id in replaced_sources & linked_surface_sources:
- linked_surface_outputs[source_id] = retained
+ for relative, owner, source_surface, surface in surface_updates:
+ original_surface = surface
+ # Generic updates already resolve each receipt at its saved source.
+ surface = projection.repair_reviewed_surface(
+ coverage,
+ projected_coverage if parent_is_canonical else {},
+ workers_by_id.get(owner),
+ source_surface,
+ surface,
+ ) or project_record("surfaces", surface, source_surface, receipts_resolved=True)
+ reconciled_surface_ids[record_key("surfaces", owner, source_surface)] = surface["id"]
+ if surface not in coverage["surfaces"] and (
+ surface is original_surface or retained_coverage_record("surfaces", surface) is None
+ ):
+ retain_coverage_row(
+ coverage["surfaces"],
+ surface,
+ owner,
+ "surfaces",
+ source_surface,
+ surface is not original_surface or relative == "parent",
+ )
for surface in coverage["surfaces"]:
if isinstance(surface, dict):
surface.setdefault("receiptRefs", [])
@@ -2508,14 +2658,25 @@ def merge_saved_results(
# Reopened work survives a superseded checkpoint, but current candidate
# outcomes still apply. Parent closures cannot remove another worker's row.
for owner, item in reopened_rows:
- if (identity := _deferred_candidate_id(item, owner, ambiguous_deferred)) is not None and (
- owner,
+ source_owner, original = record_source("deferred", owner, item)
+ if (
+ identity := _deferred_candidate_id(original, source_owner, ambiguous_deferred)
+ ) is not None and (
+ source_owner,
identity,
) in resolved:
continue
+ original_item = item
+ item = project_record("deferred", item)
pending = coverage.setdefault("deferred", [])
- if isinstance(pending, list):
- append_coverage_row(pending, item, owner, "deferred")
+ if (
+ isinstance(pending, list)
+ and item not in pending
+ and (item is original_item or retained_coverage_record("deferred", item) is None)
+ ):
+ retain_coverage_row(
+ pending, item, owner, "deferred", original_item, item is not original_item
+ )
for _, draft, owner in all_sources:
for surface in _coverage_rows(draft["coverage"], "surfaces"):
if (
@@ -2523,14 +2684,20 @@ def merge_saved_results(
and ambiguous_pending_surface(surface, owner)
and isinstance(coverage.get("surfaces"), list)
):
- retained = append_coverage_row(
- coverage["surfaces"],
- {**surface, "receiptRefs": surface.get("receiptRefs", [])},
- owner,
- "surfaces",
- )
- _inherit_coverage_sources(retained, [surface], coverage_sources)
- retained_surface_rows.add(id(surface))
+ original_surface = {**surface, "receiptRefs": surface.get("receiptRefs", [])}
+ retained_surface = project_record("surfaces", original_surface, surface)
+ if retained_surface not in coverage["surfaces"] and (
+ retained_surface is original_surface
+ or retained_coverage_record("surfaces", retained_surface) is None
+ ):
+ retain_coverage_row(
+ coverage["surfaces"],
+ retained_surface,
+ owner,
+ "surfaces",
+ surface,
+ retained_surface is not original_surface,
+ )
for relative, draft, worker_id in all_sources:
selection = source_coverage_selection(relative, worker_id)
(
@@ -2658,10 +2825,20 @@ def merge_saved_results(
if preserve_canonical_parent:
rejected_history.setdefault((worker_id, candidate_id), []).append(finding)
for item in _coverage_rows(coverage, "surfaces"):
- if (
- isinstance(item, dict)
- and item.get("candidateId") == candidate_id
- and (not preserve_canonical_parent or item.get("workerId") == worker_id)
+ if isinstance(item, dict) and (
+ coverage_candidate(None, item) == (worker_id, candidate_id)
+ or (
+ preserve_canonical_parent
+ and item.get("candidateId") == candidate_id
+ and any(
+ source_owners[source] == worker_id
+ for source in coverage_sources.get(id(item), (item, []))[1]
+ )
+ )
+ or (
+ not preserve_canonical_parent
+ and coverage_candidate(None, item) == (None, candidate_id)
+ )
):
_append_previous_finding(item, finding)
continue
@@ -2802,9 +2979,36 @@ def merge_saved_results(
continue
finding_positions[key] = (len(findings), group)
findings.append(finding)
+ worker = workers_by_id.get(worker_id)
+ # Only the accepted current output belongs to this worker attempt. An
+ # unsuperseded archive can remain evidence after that output is lost.
+ project_coverage = (
+ worker is not None
+ and worker["status"] == "succeeded"
+ and worker["result_manifest_path"] == str(scan_dir / relative)
+ and draft.get("complete") is not False
+ )
if coverage_superseded and not selected_candidates and not retain_pending:
continue
- for field in ("surfaces", "explicitExclusions", "deferred", "openQuestions"):
+ if project_coverage:
+ reviewed_attempts.add((worker_id, worker["attempt"]))
+ reviews = coverage.setdefault("reviews", [])
+ if isinstance(reviews, list) and not any(
+ isinstance(review, dict)
+ and review.get("workerId") == worker_id
+ and review.get("attempt") == worker["attempt"]
+ for review in reviews
+ ):
+ reviews.append(
+ {
+ "workerId": worker_id,
+ "attempt": worker["attempt"],
+ "completeness": draft["coverage"].get("completeness", "partial"),
+ }
+ )
+ for field in ("surfaces", "explicitExclusions", "deferred", "openQuestions", "reviews"):
+ if field == "reviews" and (worker_id is not None or field not in draft["coverage"]):
+ continue
if coverage_superseded and field not in {"surfaces", "explicitExclusions", "deferred"}:
continue
items = draft["coverage"].get(field, [])
@@ -2817,8 +3021,8 @@ def merge_saved_results(
# Keep malformed canonical collections for the existing finalizer's
# recovery and warnings rather than silently changing its contract.
continue
- for item in items:
- source_item = original_item = item
+ for index, item in enumerate(items, 1):
+ source_item = item
if not coverage_row_is_retained(field, item, worker_id, selection):
continue
if id(item) in replaced_rows.get(field, ()):
@@ -2827,7 +3031,7 @@ def merge_saved_results(
field == "deferred"
and isinstance(item, dict)
and isinstance(item.get("id"), str)
- and (worker_id, item["id"]) in closed_deferred
+ and record_key("deferred", worker_id, item) in closed_deferred
):
continue
if field == "openQuestions" and isinstance(item, str):
@@ -2842,35 +3046,51 @@ def merge_saved_results(
item = copy.deepcopy(item)
for finding in history_findings:
_append_previous_finding(item, finding, clone=True)
+ projected_row = False
+ if isinstance(item, dict):
+ original_item = item
+ item = (
+ project_missing_record(field, item, index, worker, draft["coverage"])
+ if project_coverage
+ else project_record(field, item, source_item)
+ )
+ projected_row = item is not original_item
if field == "surfaces" and isinstance(item, dict):
item = {**item, "receiptRefs": item.get("receiptRefs", [])}
+ if (
+ field == "surfaces"
+ and worker is not None
+ and isinstance(item, dict)
+ and (project_coverage or not projected_row)
+ ):
+ item = copy.deepcopy(item)
+ item["receiptRefs"] = coverage_receipts(item, worker, relative)
+ if (
+ projected_row
+ and isinstance(item, dict)
+ and retained_coverage_record(field, item) is not None
+ ):
+ continue
if isinstance(item, dict) and "id" not in item:
semantic_item = dict(item)
existing = next(
(
- row
- for row in output
- if isinstance(row, dict)
- and {key: value for key, value in row.items() if key != "id"}
+ existing
+ for existing in output
+ if isinstance(existing, dict)
+ and {key: value for key, value in existing.items() if key != "id"}
== semantic_item
),
None,
)
if existing is not None:
- if project_worker_coverage and worker_id is not None:
- worker_row_owners[id(existing)] = worker_id
- if field == "surfaces" and id(source_item) in linked_surface_sources:
- linked_surface_outputs[id(source_item)] = existing
- _inherit_coverage_sources(existing, [original_item], coverage_sources)
+ _inherit_coverage_sources(existing, [source_item], coverage_sources)
if field == "surfaces":
- retained_surface_rows.add(id(original_item))
+ retained_surface_rows.add(id(source_item))
continue
- retained = append_coverage_row(output, item, worker_id, field)
- _inherit_coverage_sources(retained, [original_item], coverage_sources)
- if field == "surfaces":
- retained_surface_rows.add(id(original_item))
- if field == "surfaces" and id(source_item) in linked_surface_sources:
- linked_surface_outputs[id(source_item)] = retained
+ retain_coverage_row(
+ output, item, worker_id, field, source_item, projected_row or project_coverage
+ )
# Generate new worker identities from the retained collection, as a normal
# Deep parent does. Authored, restored and canonical identities stay stable.
@@ -2945,212 +3165,18 @@ def merge_saved_results(
break
suffix += 1
identities.setdefault(key, []).append((variant, group, identity.get("instance")))
- recovered_worker_ids = {worker_id for _, _, worker_id in sources if worker_id is not None}
- surface_positions: dict[str, int] = {}
- resolved_surface_targets: dict[tuple[str | None, str], list[str]] = {}
-
- def remember_surface_id(item: dict[str, Any], local_id: str) -> None:
- final_id = item["id"]
- surface_positions.setdefault(final_id, len(surface_positions))
- shadowed_alias = (
- project_worker_coverage
- and id(item) not in named_surface_outputs
- and (named := worker_surfaces.get((worker_row_owners.get(id(item)), local_id)))
- is not None
- and id(named) in named_surface_outputs
- )
- for relative in coverage_sources.get(id(item), (item, []))[1]:
- # A candidate-derived ID cannot claim a link explicitly naming another surface.
- if not shadowed_alias:
- targets = surface_ids_by_source.setdefault((relative, local_id), [])
- if final_id not in targets:
- targets.append(final_id)
- candidate_id = item.get("candidateId")
- if isinstance(candidate_id, str):
- key = (source_owners[relative], candidate_id)
- if key in resolved and resolved[key] == item.get("disposition"):
- targets = resolved_surface_targets.setdefault(key, [])
- if final_id not in targets:
- targets.append(final_id)
-
- # Bind links to retained output objects before their IDs become unique.
- # Candidate metadata can give an anonymous row the same preliminary ID as
- # another surface, so only the final surface ID can be used for its links.
- retained_surface_links: dict[int, tuple[dict[str, Any], dict[str, dict[str, Any]]]] = {}
- output_tasks = _coverage_rows(coverage, "deferred")
- task_candidates = [(worker_row_owners.get(id(row)), row) for row in output_tasks]
- for task, published_id, surface, _, task_source in surface_links:
- retained_surface = linked_surface(surface, linked_surface_outputs, worker_surface_outputs)
- if retained_surface is None or not isinstance(retained_surface.get("id"), str):
- continue
- for retained_task in linked_tasks(task, published_id, task_source, task_candidates):
- _, links = retained_surface_links.setdefault(id(retained_task), (retained_task, {}))
- links.setdefault(published_id, retained_surface)
- # A newer worker checkpoint may omit a surface still needed by a retained
- # task. Restore its parent copy only when no worker replacement survived.
- for task, published_id, surface, parent_surface, task_source in surface_links:
- for retained_task in linked_tasks(task, published_id, task_source, task_candidates):
- _, links = retained_surface_links.setdefault(id(retained_task), (retained_task, {}))
- if published_id not in links and isinstance(coverage.get("surfaces"), list):
- retained_surface = append_coverage_row(
- coverage["surfaces"], parent_surface, None, "surfaces"
- )
- record_coverage_owner(
- retained_surface, surface, surface.get("workerId"), "surfaces"
- )
- links[published_id] = retained_surface
- if project_worker_coverage:
- worker_surfaces: dict[tuple[str, str], dict[str, Any]] = {}
- named_surface_outputs = {
- identity for outputs in worker_surface_outputs.values() for identity in outputs
- }
- for surface in _coverage_rows(coverage, "surfaces"):
- if not isinstance(surface, dict):
- continue
- owner, identity = worker_row_owners.get(id(surface)), surface.get("id")
- if isinstance(owner, str) and isinstance(identity, str):
- key = (owner, identity)
- existing = worker_surfaces.get(key)
- # An anonymous candidate ID can collide with an explicit surface ID.
- # Bind a named worker link to the explicitly named source.
- if existing is None or (
- id(surface) in named_surface_outputs
- and id(existing) not in named_surface_outputs
- ):
- worker_surfaces[key] = surface
- for task in output_tasks:
- if (
- not isinstance(task, dict)
- or (owner := worker_row_owners.get(id(task))) is None
- or not isinstance(task.get("surfaceIds"), list)
- ):
- continue
- for identity in task["surfaceIds"]:
- if (
- isinstance(identity, str)
- and (surface := worker_surfaces.get((owner, identity))) is not None
- ):
- _, links = retained_surface_links.setdefault(id(task), (task, {}))
- links.setdefault(identity, surface)
- for field in ("surfaces", "explicitExclusions", "deferred"):
- used: set[str] = set()
- items = coverage.setdefault(field, [])
- if not isinstance(items, list):
- continue
- reserved = {
- row["id"] for row in items if isinstance(row, dict) and isinstance(row.get("id"), str)
- }
- normalized_items = []
- for item in items:
- normalized_items.append(item)
- if not isinstance(item, dict):
- continue
- if id(item) in canonical_rows:
- if isinstance(item.get("id"), str):
- used.add(item["id"])
- if field == "surfaces":
- remember_surface_id(item, item["id"])
- continue
- item.setdefault("id", _saved_coverage_id(item))
- if not isinstance(item["id"], str):
- # Preserve malformed rows for per-record recovery, including frozen replay.
- continue
- local_id = item["id"]
- scoped = (
- preserve_canonical_parent
- and isinstance(worker_id := item.get("workerId"), str)
- and worker_id in recovered_worker_ids
- )
- if scoped:
- # Resolve worker-local candidate/deferred history before scoping IDs.
- item["id"] = f"{worker_id}/{item['id']}"
- if isinstance(item.get("surfaceIds"), list):
- links: list[Any] = []
- for surface_id in item["surfaceIds"]:
- targets: list[str] = []
- declared = False
- if field == "deferred" and isinstance(surface_id, str):
- for relative in coverage_sources.get(id(item), (item, []))[1]:
- source_key = (relative, surface_id)
- history_keys = [source_key]
- if source_key not in surface_ids_by_source:
- # A deferred-only checkpoint can refer to an earlier
- # observation from this logical worker, including retries.
- # Reused IDs can denote several retained observations.
- history_keys = [
- (historical, surface_id)
- for historical, _, owner in all_sources
- if owner == source_owners[relative]
- and source_order[historical] <= source_order[relative]
- ]
- for key in history_keys:
- declared |= key in surface_ids_by_source
- for target in surface_ids_by_source.get(key, []):
- if target not in targets:
- targets.append(target)
- if targets:
- targets.sort(key=surface_positions.__getitem__)
- for target in targets:
- if target not in links:
- links.append(target)
- elif not declared:
- links.append(
- f"{worker_id}/{surface_id}"
- if scoped and isinstance(surface_id, str)
- else surface_id
- )
- item["surfaceIds"] = links
- if scoped:
- if item in normalized_items[:-1]:
- # A previous finish may have published files before its DB commit.
- if field == "surfaces":
- remember_surface_id(item, local_id)
- normalized_items.pop()
- continue
- if item["id"] in used:
- if project_worker_coverage:
- base = item["id"]
- suffix = 2
- while item["id"] in used or item["id"] in reserved:
- item["id"] = f"{base}-{suffix}"
- suffix += 1
- else:
- item["id"] = f"{item['id']}-{_digest(item)[:16]}"
- if preserve_canonical_parent and item in normalized_items[:-1]:
- if field == "surfaces":
- remember_surface_id(item, local_id)
- normalized_items.pop()
- continue
- used.add(item["id"])
- if field == "surfaces":
- item.setdefault("receiptRefs", [])
- remember_surface_id(item, local_id)
- if preserve_canonical_parent and isinstance(items, list):
- coverage[field] = normalized_items
- if field == "surfaces":
- # Candidate resolution replaces pending surfaces independently of generic
- # tasks. Preserve their original links, even when the outcome changes ID
- # or a distinct retained surface shares the old source-local ID.
- for relative, draft, owner in all_sources:
- originals = draft["coverage"].get("surfaces", [])
- for original in originals if isinstance(originals, list) else []:
- if (
- not isinstance(original, dict)
- or id(original) in retained_surface_rows
- or original.get("disposition") != "needs_follow_up"
- or not isinstance(local_id := original.get("id"), str)
- or not isinstance(candidate_id := original.get("candidateId"), str)
- ):
- continue
- targets = surface_ids_by_source.setdefault((relative, local_id), [])
- for target in resolved_surface_targets.get((owner, candidate_id), []):
- if target not in targets:
- targets.append(target)
- for task, links in retained_surface_links.values():
- task["surfaceIds"] = [
- links[identity]["id"] if isinstance(identity, str) and identity in links else identity
- for identity in task["surfaceIds"]
- ]
+ projection.finalize(
+ coverage,
+ canonical_rows,
+ _digest,
+ preserve_canonical_parent=preserve_canonical_parent,
+ all_sources=all_sources,
+ source_order=source_order,
+ coverage_sources=coverage_sources,
+ surface_ids_by_source=surface_ids_by_source,
+ retained_surface_rows=retained_surface_rows,
+ resolved=resolved,
+ )
coverage.pop("completeness", None)
if stopped:
if not isinstance(coverage.get("deferred"), list):
@@ -3158,4 +3184,8 @@ def merge_saved_results(
item = {"id": "scan-stopped", "reason": reason}
if item not in coverage["deferred"]:
coverage["deferred"].append(item)
+ surfaces = coverage.get("surfaces", [])
+ for surface in surfaces if isinstance(surfaces, list) else []:
+ if isinstance(surface, dict) and isinstance(surface.get("provenance"), dict):
+ surface["provenance"].pop("scanReceiptRefs", None)
return manifest, {"findings": findings}, coverage
diff --git a/plugins/codex-security/scripts/workbench/saved_coverage.py b/plugins/codex-security/scripts/workbench/saved_coverage.py
index d4a439ee9..f83ebbfb4 100644
--- a/plugins/codex-security/scripts/workbench/saved_coverage.py
+++ b/plugins/codex-security/scripts/workbench/saved_coverage.py
@@ -11,7 +11,10 @@ from typing import Any
from finalize_scan_contract import (
ContractError,
_normalize_unsealed_open_questions,
+ _read_json,
_require_portable_relative_path,
+ _validate_resolved_deferred,
+ _validate_schema_node,
)
@@ -98,6 +101,7 @@ def _prepare_worker_coverage_recovery(
source_order: dict[str, tuple[int, int]],
*,
enabled: bool,
+ receipt_resolver=None,
) -> tuple[
bool, list[tuple[dict[str, Any], str, dict[str, Any], dict[str, Any], dict[str, Any] | None]]
]:
@@ -156,7 +160,9 @@ def _prepare_worker_coverage_recovery(
if not isinstance(row, dict):
continue
row["workerId"] = owner
- if field == "surfaces" and isinstance(row.get("receiptRefs"), list):
+ if field == "surfaces" and receipt_resolver is not None:
+ row["receiptRefs"] = receipt_resolver(relative, row, owner)
+ elif field == "surfaces" and isinstance(row.get("receiptRefs"), list):
refs = []
for ref in row["receiptRefs"]:
if isinstance(ref, str):
@@ -232,3 +238,635 @@ def _prepare_worker_coverage_recovery(
if _coverage_comparison_row(row, field) not in projected[field]
]
return True, surface_links
+
+
+class SavedCoverageProjection:
+ """Keep worker source links stable while legacy and accepted rows converge."""
+
+ def __init__(self, project_worker_coverage, legacy_surface_links, sources):
+ self.project_worker_coverage = project_worker_coverage
+ self.legacy_surface_links = legacy_surface_links
+ self.legacy_source_outputs: dict[int, dict[str, Any]] = {}
+ self.worker_row_owners: dict[int, str] = {}
+ self.worker_surface_outputs: dict[tuple[str, str], dict[int, dict[str, Any]]] = {}
+ self.source_owner_by_id = {
+ id(row): owner
+ for _, draft, owner in sources
+ for field in ("surfaces", "deferred")
+ for rows in (draft["coverage"].get(field, []),)
+ if isinstance(rows, list)
+ for row in rows
+ if isinstance(row, dict)
+ }
+
+ def record_coverage_owner(self, retained, original, owner, field):
+ if self.project_worker_coverage and owner is not None and isinstance(retained, dict):
+ self.worker_row_owners[id(retained)] = owner
+ if field == "surfaces" and isinstance(original.get("id"), str):
+ self.worker_surface_outputs.setdefault((owner, original["id"]), {})[
+ id(retained)
+ ] = retained
+
+ def retain_coverage_row(self, output, item, owner, field, original=None, qualified=False):
+ retained = (
+ copy.deepcopy(item)
+ if qualified or not self.project_worker_coverage
+ else _project_worker_coverage_row(item, owner, field)
+ )
+ if retained not in output:
+ output.append(retained)
+ retained = output[output.index(retained)]
+ if isinstance(retained, dict) and original is not None:
+ self.legacy_source_outputs[id(original)] = retained
+ self.record_coverage_owner(
+ retained, original if original is not None else item, owner, field
+ )
+ return retained
+
+ def retained_record(self, coverage, field, item, canonical_record):
+ item = canonical_record(field, item)
+ if field == "surfaces":
+ # Canonical IDs can change while source content and ownership stay the same.
+ item = {key: value for key, value in item.items() if key != "id"}
+ records = coverage.get(field, [])
+ for record in records if isinstance(records, list) else []:
+ if not isinstance(record, dict):
+ continue
+ original = dict(canonical_record(field, record))
+ if "id" not in item:
+ original.pop("id", None) # Canonical publication can assign an ID.
+ if field == "openQuestions":
+ # Compare the same canonical form that publication writes.
+ previous_question = {"openQuestions": [original]}
+ source_question = {"openQuestions": [item]}
+ _normalize_unsealed_open_questions(previous_question)
+ _normalize_unsealed_open_questions(source_question)
+ if previous_question == source_question:
+ return record
+ elif original == item:
+ return record
+ return None
+
+ def linked_surface(self, surface, outputs, named_outputs):
+ retained = outputs.get(id(surface))
+ owner = self.source_owner_by_id.get(id(surface), surface.get("workerId"))
+ if retained is None and isinstance(owner, str) and isinstance(surface.get("id"), str):
+ matches = named_outputs.get((owner, surface["id"]), {})
+ if len(matches) == 1:
+ retained = next(iter(matches.values()))
+ return retained
+
+ def is_reviewed_worker(self, canonical, worker):
+ if (
+ worker is None
+ or not {"status", "merge_state", "attempt"}.issubset(worker.keys())
+ or worker["status"] != "succeeded"
+ or worker["merge_state"] != "merged"
+ ):
+ return False
+ reviews = canonical.get("reviews", [])
+ return any(
+ isinstance(review, dict)
+ and review.get("workerId") == worker["id"]
+ and review.get("attempt") == worker["attempt"]
+ for review in (reviews if isinstance(reviews, list) else [])
+ )
+
+ def repair_reviewed_surface(self, coverage, canonical, worker, source, surface):
+ if not self.is_reviewed_worker(canonical, worker):
+ return None
+ published = next(
+ (
+ row
+ for row in canonical.get("surfaces", [])
+ if isinstance(row, dict)
+ and isinstance(provenance := row.get("provenance"), dict)
+ and provenance.get("workerId") == worker["id"]
+ and provenance.get("attempt") == worker["attempt"]
+ and provenance.get("sourceId") == surface.get("id")
+ and row.get("disposition") == surface.get("disposition")
+ ),
+ None,
+ )
+ if published is None:
+ return None
+ retained = next(
+ (
+ row
+ for row in coverage["surfaces"]
+ if row.get("id") == published.get("id")
+ and row.get("provenance") == published.get("provenance")
+ ),
+ None,
+ )
+ if retained is None:
+ retained = copy.deepcopy(published)
+ # Receipt repair can recover earlier evidence without replacing the
+ # accepted surface's identity, disposition, or host-owned provenance.
+ retained["receiptRefs"] = surface["receiptRefs"]
+ return self.retain_coverage_row(
+ coverage["surfaces"], retained, worker["id"], "surfaces", source, True
+ )
+
+ def linked_tasks(self, task, published_id, source_task, candidates):
+ expected = None
+ if source_task is not None:
+ expected = self.legacy_source_outputs.get(id(source_task))
+ if expected is None:
+ expected = _project_worker_coverage_row(
+ source_task, self.source_owner_by_id.get(id(source_task)), "deferred"
+ )
+ return [
+ retained
+ for owner, retained in candidates
+ if retained == task
+ or (
+ expected is not None
+ and (
+ retained is expected
+ or (
+ owner == self.source_owner_by_id.get(id(source_task))
+ and isinstance(retained, dict)
+ and retained.get("id") == expected.get("id")
+ and isinstance(retained.get("surfaceIds"), list)
+ and published_id in retained["surfaceIds"]
+ )
+ )
+ )
+ ]
+
+ def finalize(
+ self,
+ coverage,
+ canonical_rows,
+ digest,
+ *,
+ preserve_canonical_parent,
+ all_sources,
+ source_order,
+ coverage_sources,
+ surface_ids_by_source,
+ retained_surface_rows,
+ resolved,
+ ):
+ from workbench.retained_coverage import _inherit_coverage_sources
+
+ source_owners = {relative: owner for relative, _, owner in all_sources}
+ recovered_worker_ids = {
+ worker_id for _, _, worker_id in all_sources if worker_id is not None
+ }
+ surface_positions: dict[str, int] = {}
+ resolved_surface_targets: dict[tuple[str | None, str], list[str]] = {}
+
+ def remember_surface_id(item: dict[str, Any], local_id: str) -> None:
+ final_id = item["id"]
+ surface_positions.setdefault(final_id, len(surface_positions))
+ shadowed_alias = (
+ self.project_worker_coverage
+ and id(item) not in named_surface_outputs
+ and (named := worker_surfaces.get((self.worker_row_owners.get(id(item)), local_id)))
+ is not None
+ and id(named) in named_surface_outputs
+ )
+ for relative in coverage_sources.get(id(item), (item, []))[1]:
+ # A candidate-derived ID cannot claim a link explicitly naming another surface.
+ if not shadowed_alias:
+ targets = surface_ids_by_source.setdefault((relative, local_id), [])
+ if final_id not in targets:
+ targets.append(final_id)
+ candidate_id = item.get("candidateId")
+ if isinstance(candidate_id, str):
+ key = (source_owners[relative], candidate_id)
+ if key in resolved and resolved[key] == item.get("disposition"):
+ targets = resolved_surface_targets.setdefault(key, [])
+ if final_id not in targets:
+ targets.append(final_id)
+
+ retained_legacy_surface_links: dict[
+ int, tuple[dict[str, Any], dict[str, dict[str, Any]]]
+ ] = {}
+ output_tasks = coverage.get("deferred", [])
+ task_candidates = [
+ (self.worker_row_owners.get(id(row)), row)
+ for row in (output_tasks if isinstance(output_tasks, list) else [])
+ ]
+ for task, published_id, surface, _, task_source in self.legacy_surface_links:
+ retained_surface = self.linked_surface(
+ surface, self.legacy_source_outputs, self.worker_surface_outputs
+ )
+ if retained_surface is None or not isinstance(retained_surface.get("id"), str):
+ continue
+ for retained_task in self.linked_tasks(
+ task, published_id, task_source, task_candidates
+ ):
+ _, links = retained_legacy_surface_links.setdefault(
+ id(retained_task), (retained_task, {})
+ )
+ links.setdefault(published_id, retained_surface)
+ # A newer worker checkpoint may omit a surface still needed by a retained
+ # task. Restore its parent copy only when no worker replacement survived.
+ for task, published_id, surface, parent_surface, task_source in self.legacy_surface_links:
+ for retained_task in self.linked_tasks(
+ task, published_id, task_source, task_candidates
+ ):
+ _, links = retained_legacy_surface_links.setdefault(
+ id(retained_task), (retained_task, {})
+ )
+ if published_id not in links and isinstance(coverage.get("surfaces"), list):
+ retained_surface = self.retain_coverage_row(
+ coverage["surfaces"], parent_surface, None, "surfaces"
+ )
+ self.record_coverage_owner(
+ retained_surface,
+ surface,
+ self.source_owner_by_id.get(id(surface), surface.get("workerId")),
+ "surfaces",
+ )
+ _inherit_coverage_sources(retained_surface, [parent_surface], coverage_sources)
+ links[published_id] = retained_surface
+ if self.project_worker_coverage:
+ worker_surfaces: dict[tuple[str, str], dict[str, Any]] = {}
+ named_surface_outputs = {
+ identity for outputs in self.worker_surface_outputs.values() for identity in outputs
+ }
+ surfaces = coverage.get("surfaces", [])
+ for surface in surfaces if isinstance(surfaces, list) else []:
+ if not isinstance(surface, dict):
+ continue
+ owner, identity = self.worker_row_owners.get(id(surface)), surface.get("id")
+ if isinstance(owner, str) and isinstance(identity, str):
+ key = (owner, identity)
+ existing = worker_surfaces.get(key)
+ # An anonymous candidate ID can collide with an explicit surface ID.
+ # Bind a named worker link to the explicitly named source.
+ if existing is None or (
+ id(surface) in named_surface_outputs
+ and id(existing) not in named_surface_outputs
+ ):
+ worker_surfaces[key] = surface
+ for task in output_tasks if isinstance(output_tasks, list) else []:
+ if (
+ not isinstance(task, dict)
+ or (owner := self.worker_row_owners.get(id(task))) is None
+ or not isinstance(task.get("surfaceIds"), list)
+ ):
+ continue
+ for identity in task["surfaceIds"]:
+ if (
+ isinstance(identity, str)
+ and (surface := worker_surfaces.get((owner, identity))) is not None
+ ):
+ _, links = retained_legacy_surface_links.setdefault(id(task), (task, {}))
+ links.setdefault(identity, surface)
+ for field in ("surfaces", "explicitExclusions", "deferred"):
+ used: set[str] = set()
+ items = coverage.setdefault(field, [])
+ reserved = {
+ row["id"]
+ for row in (items if isinstance(items, list) else [])
+ if isinstance(row, dict) and isinstance(row.get("id"), str)
+ }
+ normalized_items = []
+ for item in items if isinstance(items, list) else []:
+ normalized_items.append(item)
+ if not isinstance(item, dict):
+ continue
+ if id(item) in canonical_rows:
+ if isinstance(item.get("id"), str):
+ used.add(item["id"])
+ if field == "surfaces":
+ remember_surface_id(item, item["id"])
+ continue
+ item.setdefault("id", _saved_coverage_id(item))
+ if not isinstance(item["id"], str):
+ # Preserve malformed rows for per-record recovery, including frozen replay.
+ continue
+ local_id = item["id"]
+ scoped = (
+ preserve_canonical_parent
+ and isinstance(worker_id := item.get("workerId"), str)
+ and worker_id in recovered_worker_ids
+ )
+ if scoped:
+ # Resolve worker-local candidate/deferred history before scoping IDs.
+ item["id"] = f"{worker_id}/{item['id']}"
+ if isinstance(item.get("surfaceIds"), list):
+ links: list[Any] = []
+ for surface_id in item["surfaceIds"]:
+ targets: list[str] = []
+ declared = False
+ if field == "deferred" and isinstance(surface_id, str):
+ for relative in coverage_sources.get(id(item), (item, []))[1]:
+ source_key = (relative, surface_id)
+ history_keys = [source_key]
+ if source_key not in surface_ids_by_source:
+ # A deferred-only checkpoint can refer to an earlier
+ # observation from this logical worker, including retries.
+ # Reused IDs can denote several retained observations.
+ history_keys = [
+ (historical, surface_id)
+ for historical, _, owner in all_sources
+ if owner == source_owners[relative]
+ and source_order[historical] <= source_order[relative]
+ ]
+ for key in history_keys:
+ declared |= key in surface_ids_by_source
+ for target in surface_ids_by_source.get(key, []):
+ if target not in targets:
+ targets.append(target)
+ if targets:
+ targets.sort(key=surface_positions.__getitem__)
+ for target in targets:
+ if target not in links:
+ links.append(target)
+ elif not declared:
+ links.append(
+ f"{worker_id}/{surface_id}"
+ if scoped and isinstance(surface_id, str)
+ else surface_id
+ )
+ item["surfaceIds"] = links
+ if scoped:
+ if preserve_canonical_parent and item in normalized_items[:-1]:
+ # A previous finish may have published files before its DB commit.
+ if field == "surfaces":
+ remember_surface_id(item, local_id)
+ normalized_items.pop()
+ continue
+ if item["id"] in used:
+ if self.project_worker_coverage:
+ base = item["id"]
+ suffix = 2
+ while item["id"] in used or item["id"] in reserved:
+ item["id"] = f"{base}-{suffix}"
+ suffix += 1
+ else:
+ item["id"] = f"{item['id']}-{digest(item)[:16]}"
+ if preserve_canonical_parent and item in normalized_items[:-1]:
+ if field == "surfaces":
+ remember_surface_id(item, local_id)
+ normalized_items.pop()
+ continue
+ used.add(item["id"])
+ if field == "surfaces":
+ item.setdefault("receiptRefs", [])
+ remember_surface_id(item, local_id)
+ if preserve_canonical_parent and isinstance(items, list):
+ coverage[field] = normalized_items
+ if field == "surfaces":
+ # Candidate resolution replaces pending surfaces independently of generic
+ # tasks. Preserve their original links, even when the outcome changes ID
+ # or a distinct retained surface shares the old source-local ID.
+ for relative, draft, owner in all_sources:
+ originals = draft["coverage"].get("surfaces", [])
+ for original in originals if isinstance(originals, list) else []:
+ if (
+ not isinstance(original, dict)
+ or id(original) in retained_surface_rows
+ or original.get("disposition") != "needs_follow_up"
+ or not isinstance(local_id := original.get("id"), str)
+ or not isinstance(candidate_id := original.get("candidateId"), str)
+ ):
+ continue
+ targets = surface_ids_by_source.setdefault((relative, local_id), [])
+ for target in resolved_surface_targets.get((owner, candidate_id), []):
+ if target not in targets:
+ targets.append(target)
+ for task, links in retained_legacy_surface_links.values():
+ task["surfaceIds"] = [
+ links[identity]["id"]
+ if isinstance(identity, str) and identity in links
+ else identity
+ for identity in task["surfaceIds"]
+ ]
+
+ def fallback_surfaces(
+ self,
+ coverage,
+ all_sources,
+ deferred_rows,
+ reopened_rows,
+ *,
+ selection_for_source,
+ row_is_retained,
+ replaced_deferred,
+ closed_deferred,
+ ambiguous_surface_ids,
+ deferred_is_resolved,
+ ):
+ fallback_surfaces = []
+ if self.legacy_surface_links and isinstance(coverage.get("surfaces"), list):
+ # Plan the surviving associations before historical receipts can claim
+ # a spelling needed by a surface that must be restored for its task.
+ prospective_tasks = []
+ prospective_surfaces = {}
+ prospective_named = {}
+ for relative, draft, owner in all_sources:
+ selection = selection_for_source(relative, owner)
+ for row in deferred_rows[relative]:
+ if (
+ row_is_retained("deferred", row, owner, selection)
+ and id(row) not in replaced_deferred
+ and not (
+ isinstance(row, dict)
+ and isinstance(row.get("id"), str)
+ and (owner, row["id"]) in closed_deferred
+ )
+ ):
+ prospective_tasks.append(
+ (owner, _project_worker_coverage_row(row, owner, "deferred"))
+ )
+ rows = draft["coverage"].get("surfaces", [])
+ for row in rows if isinstance(rows, list) else []:
+ if owner is None or not isinstance(row, dict):
+ continue
+ emitted = row_is_retained("surfaces", row, owner, selection)
+ ambiguous = (
+ isinstance(row.get("id"), str)
+ and (owner, row["id"]) in ambiguous_surface_ids
+ and row.get("disposition") == "needs_follow_up"
+ and not any(key in row for key in ("candidateId", "candidate", "finding"))
+ )
+ if not emitted and not ambiguous:
+ continue
+ projected = _project_worker_coverage_row(row, owner, "surfaces")
+ if isinstance(row.get("id"), str):
+ named = prospective_named.setdefault((owner, row["id"]), {})
+ projected = next(
+ (existing for existing in named.values() if existing == projected),
+ projected,
+ )
+ named[id(projected)] = projected
+ if emitted:
+ prospective_surfaces[id(row)] = projected
+ prospective_tasks.extend(
+ (owner, _project_worker_coverage_row(row, owner, "deferred"))
+ for owner, row in reopened_rows
+ if not deferred_is_resolved(owner, row)
+ )
+ bound = {
+ (id(retained), published_id)
+ for task, published_id, surface, _, task_source in self.legacy_surface_links
+ if self.linked_surface(surface, prospective_surfaces, prospective_named) is not None
+ for retained in self.linked_tasks(
+ task, published_id, task_source, prospective_tasks
+ )
+ }
+ for task, published_id, _, parent_surface, task_source in self.legacy_surface_links:
+ for retained in self.linked_tasks(
+ task, published_id, task_source, prospective_tasks
+ ):
+ key = (id(retained), published_id)
+ if key not in bound:
+ fallback_surfaces.append(parent_surface)
+ bound.add(key)
+ return fallback_surfaces
+
+
+def prepare_saved_coverage_projection(
+ sources,
+ parent,
+ source_order,
+ *,
+ project_worker_coverage,
+ receipt_resolver,
+ preserve_canonical_parent=False,
+):
+ if preserve_canonical_parent:
+ return SavedCoverageProjection(False, [], sources)
+ # Older parent publications use worker/id identities. Reconcile those copies
+ # against their bound sources before projecting accepted attempt provenance.
+ projected_metadata = {
+ id(row): ("workerId" in row, row.get("workerId"))
+ for relative, draft, _ in sources
+ for field in ("surfaces", "explicitExclusions", "deferred", "openQuestions")
+ for rows in (draft["coverage"].get(field, []),)
+ if isinstance(rows, list)
+ for row in rows
+ if isinstance(row, dict)
+ }
+ project_worker_coverage, legacy_surface_links = _prepare_worker_coverage_recovery(
+ sources,
+ parent,
+ source_order,
+ enabled=project_worker_coverage,
+ receipt_resolver=receipt_resolver,
+ )
+ for _, draft, _ in sources:
+ for field in ("surfaces", "explicitExclusions", "deferred", "openQuestions"):
+ rows = draft["coverage"].get(field, [])
+ if not isinstance(rows, list):
+ continue
+ for row in rows:
+ if not isinstance(row, dict):
+ continue
+ present, value = projected_metadata.get(id(row), (False, None))
+ if present:
+ row["workerId"] = value
+ else:
+ row.pop("workerId", None)
+ return SavedCoverageProjection(project_worker_coverage, legacy_surface_links, sources)
+
+
+def _deferred_rows(coverage: dict[str, Any]) -> list[Any]:
+ rows = coverage.get("deferred", [])
+ return rows if isinstance(rows, list) else []
+
+
+def _resolved_deferred_rows(draft: dict[str, Any], schema: dict[str, Any]) -> list[dict[str, Any]]:
+ if draft.get("complete") is False:
+ return []
+ coverage = draft["coverage"]
+ rows = coverage.get("resolvedDeferred", [])
+ try:
+ # Invalid closure metadata cannot discard the evidence it names.
+ _validate_schema_node(rows, schema, "coverage.resolvedDeferred")
+ _validate_resolved_deferred({**coverage, "deferred": _deferred_rows(coverage)})
+ except ContractError:
+ return []
+ return rows
+
+
+def _project_worker_closure(
+ relative: str,
+ owner: str,
+ closure: dict[str, Any],
+ worker_attempts: dict[str, tuple[str, int]],
+ source_order: dict[str, tuple[int, int]],
+) -> dict[str, Any]:
+ directory = Path(relative).parent
+ if directory.name == "checkpoints":
+ directory = directory.parent
+ _, attempt = worker_attempts.get(directory.as_posix(), (owner, source_order[relative][0]))
+ return {
+ "id": f"{owner}-attempt-{attempt}-resolved-{closure['id']}",
+ "reason": closure["reason"],
+ }
+
+
+def _projected_worker_closure_sources(
+ sources: list[tuple[str, dict[str, Any], str | None]],
+ source_order: dict[str, tuple[int, int]],
+ worker_attempts: dict[str, tuple[str, int]],
+ schema: dict[str, Any],
+) -> dict[tuple[str, str], tuple[str, dict[str, Any], str]]:
+ projected: dict[tuple[str, str], tuple[str, dict[str, Any], str]] = {}
+ for relative, draft, owner in sources:
+ if owner is None:
+ continue
+ for closure in _resolved_deferred_rows(draft, schema):
+ row = _project_worker_closure(relative, owner, closure, worker_attempts, source_order)
+ # Both fields identify the full strict closeout row, including its reason.
+ key = (row["id"], row["reason"])
+ previous = projected.get(key)
+ if previous is None or source_order[relative] > source_order[previous[2]]:
+ projected[key] = (owner, closure, relative)
+ return projected
+
+
+def _parent_observation_content(parent: dict[str, Any]) -> dict[str, Any]:
+ # Frozen recovery ancestry does not change the observed scan result.
+ return {
+ key: value
+ for key, value in parent.items()
+ if key not in {"previousParentCheckpoints", "canonicalParent"}
+ }
+
+
+def _merge_tied_parent_observations(
+ current: dict[str, Any], previous: dict[str, Any]
+) -> dict[str, Any]:
+ merged = copy.deepcopy(current)
+ for finding in previous["findings"]:
+ if finding not in merged["findings"]:
+ merged["findings"].append(copy.deepcopy(finding))
+ coverage = merged["coverage"]
+ for field in ("surfaces", "explicitExclusions", "deferred", "openQuestions"):
+ rows = previous["coverage"].get(field, [])
+ output = coverage.setdefault(field, [])
+ if isinstance(rows, list) and isinstance(output, list):
+ for row in rows:
+ if row not in output:
+ output.append(copy.deepcopy(row))
+ pending_ids = {
+ identity
+ for row in _deferred_rows(coverage)
+ if isinstance(row, dict)
+ for identity in (row.get("id"), row.get("candidateId"))
+ if isinstance(identity, str)
+ }
+ closure_schema = _read_json(
+ Path(__file__).resolve().parents[2] / "schemas" / "coverage.schema.json"
+ )["properties"]["resolvedDeferred"]
+ closures = {}
+ for observed in (current, previous):
+ for row in _resolved_deferred_rows(observed, closure_schema):
+ if row["id"] not in pending_ids:
+ closures.setdefault(row["id"], copy.deepcopy(row))
+ coverage.pop("resolvedDeferred", None)
+ if closures:
+ coverage["resolvedDeferred"] = list(closures.values())
+ if current.get("complete") is False or previous.get("complete") is False:
+ merged["complete"] = False
+ coverage.pop("completeness", None)
+ return merged
diff --git a/plugins/codex-security/scripts/workbench/saved_sources.py b/plugins/codex-security/scripts/workbench/saved_sources.py
index dce2fd9af..70e4ca149 100644
--- a/plugins/codex-security/scripts/workbench/saved_sources.py
+++ b/plugins/codex-security/scripts/workbench/saved_sources.py
@@ -9,6 +9,7 @@ import os
import re
import stat
from collections.abc import Iterator
+from functools import partial
from pathlib import Path
from typing import Any
@@ -19,6 +20,7 @@ from finalize_scan_contract import (
manifest_is_sealed,
write_scan_local_bytes,
)
+from workbench_coverage_receipts import capture_receipt_checkpoints
def _encoded(value: Any) -> bytes:
@@ -99,7 +101,13 @@ def _capture_saved_source(
) -> dict[str, tuple[str, int]]:
if not snapshot_head or Path(relative).name != "checkpoint-head.json":
_, digest, observed = _read_saved_result(scan_dir, relative, scan_id, kind=kind)
- return {relative: (digest, observed)}
+ return capture_receipt_checkpoints(
+ relative,
+ kind,
+ {relative: (digest, observed)},
+ partial(_checkpoint_paths, scan_dir),
+ partial(_read_saved_result, scan_dir, scan_id=scan_id),
+ )
head, _, observed = _read_saved_result(scan_dir, relative, scan_id)
observation = {
**{key: head[key] for key in ("checkpoint", "inputCheckpoint") if key in head},
diff --git a/plugins/codex-security/scripts/workbench_coverage_receipts.py b/plugins/codex-security/scripts/workbench_coverage_receipts.py
new file mode 100644
index 000000000..50112d0b1
--- /dev/null
+++ b/plugins/codex-security/scripts/workbench_coverage_receipts.py
@@ -0,0 +1,479 @@
+"""Resolve accepted worker receipt references without changing their ownership."""
+
+from __future__ import annotations
+
+import copy
+import os
+import re
+from collections.abc import Callable, Sequence
+from pathlib import Path, PurePosixPath
+from typing import Any
+
+from finalize_scan_contract import (
+ ContractError,
+ _require_portable_relative_path,
+ _require_str,
+ _validate_schema_node,
+ open_scan_local_file_descriptor,
+ path_within_scope,
+)
+from workbench.saved_coverage import _project_worker_coverage_row, _saved_coverage_id
+
+
+def _checkpoint_head_directory(relative: str) -> Path | None:
+ path = Path(relative)
+ if path.name == "checkpoint-head.json":
+ return path.parent
+ if path.parent.name == "checkpoint-heads":
+ return path.parent.parent
+ return None
+
+
+def _is_source_order_snapshot(relative: str) -> bool:
+ path = Path(relative)
+ return path.parent == Path("source-order") and bool(
+ re.fullmatch(r"[0-9a-f]{64}\.json", path.name)
+ )
+
+
+def capture_receipt_checkpoints(
+ relative: str,
+ kind: str | None,
+ captured: dict[str, tuple[str, int]],
+ checkpoint_paths: Callable[[str], list[str]],
+ read_checkpoint: Callable[[str], tuple[dict[str, Any], str, int]],
+) -> dict[str, tuple[str, int]]:
+ """Bind a captured worker result to checkpoints visible after reading it."""
+ if kind == "discovery" and Path(relative).name == "result.json":
+ directory = (Path(relative).parent / "checkpoints").as_posix()
+ for path in checkpoint_paths(directory):
+ try:
+ _, digest, observed = read_checkpoint(path)
+ except (ContractError, OSError, ValueError):
+ continue
+ captured[path] = (digest, observed)
+ return captured
+
+
+def receipt_checkpoints(
+ relative: str,
+ worker_id: str | None,
+ known: dict[str, str | None],
+ checkpoint_paths: Callable[[str], list[str]],
+) -> dict[str, str]:
+ """Capture host checkpoints written after enumeration but before result.json."""
+ if worker_id is None or Path(relative).name != "result.json":
+ return {}
+ directory = (Path(relative).parent / "checkpoints").as_posix()
+ added = {path: worker_id for path in checkpoint_paths(directory) if path not in known}
+ known.update(added)
+ return added
+
+
+def retain_recorded_receipt_ownership(
+ sources: list[tuple[str, dict[str, Any], str | None]],
+ digest: Callable[[Any], str],
+ include_paths: Sequence[str],
+ finding_scope: Sequence[tuple[str, bool]] | None = None,
+) -> None:
+ # Checkpoints are written by the host tool; direct result files can be authored
+ # without it. Compare the entire payload within the same worker attempt.
+ def payload_digest(draft: dict[str, Any]) -> str:
+ return digest({key: value for key, value in draft.items() if key != "handoffClaimToken"})
+
+ scope = (
+ finding_scope if finding_scope is not None else [(path, False) for path in include_paths]
+ )
+ recorded: dict[Path, set[str]] = {}
+ for relative, draft, worker_id in sources:
+ directory = Path(relative).parent
+ if worker_id is not None and directory.name == "checkpoints":
+ payloads = recorded.setdefault(directory.parent, set())
+ payloads.add(payload_digest(draft))
+ if "." not in include_paths:
+ # Legacy writers saved the checkpoint before filtering inherited findings.
+ scoped = [
+ finding
+ for finding in draft["findings"]
+ if isinstance(finding, dict)
+ and isinstance(finding.get("locations"), list)
+ and any(
+ isinstance(location, dict)
+ and isinstance(location.get("path"), str)
+ and any(
+ PurePosixPath(location["path"]) == PurePosixPath(path)
+ if is_file
+ else path_within_scope(location["path"], path)
+ for path, is_file in scope
+ )
+ for location in finding["locations"]
+ )
+ ]
+ payloads.add(payload_digest({**draft, "findings": scoped}))
+ for relative, draft, worker_id in sources:
+ directory = Path(relative).parent
+ if worker_id is None or directory.name == "checkpoints":
+ continue
+ if payload_digest(draft) in recorded.get(directory, set()):
+ continue
+ surfaces = draft["coverage"].get("surfaces", [])
+ for surface in surfaces if isinstance(surfaces, list) else []:
+ if isinstance(surface, dict) and isinstance(surface.get("provenance"), dict):
+ surface["provenance"].pop("scanReceiptRefs", None)
+
+
+def resolve_coverage_receipts(
+ scan_dir: Path, item: dict[str, Any], worker: Any, relative: str
+) -> Any:
+ refs = item.get("receiptRefs", [])
+ if not isinstance(refs, list):
+ return refs
+ directory = Path(relative).parent
+ if directory.name == "checkpoints":
+ directory = directory.parent
+ output = Path(worker["artifact_dir"]).relative_to(scan_dir)
+ worker_prefix = (output.parent if output.name == "output" else output).as_posix() + "/"
+ active_prefix = output.as_posix() + "/"
+ if directory != output:
+ refs = [
+ f"{directory.as_posix()}/{Path(ref).as_posix()[len(active_prefix) :]}"
+ if isinstance(ref, str) and Path(ref).as_posix().startswith(active_prefix)
+ else ref
+ for ref in refs
+ ]
+
+ def scan_receipt(ref: str) -> bool:
+ try:
+ descriptor = open_scan_local_file_descriptor(scan_dir, ref, "coverage receipt")
+ except (ContractError, OSError):
+ return False
+ os.close(descriptor)
+ return True
+
+ def qualified_receipt(ref: Any) -> Any:
+ if not isinstance(ref, str):
+ return ref
+ try:
+ normalized = _require_portable_relative_path(ref, "Worker coverage receipt")
+ except ContractError:
+ return ref # Leave malformed references to per-record recovery.
+ if not normalized.startswith("artifacts/"):
+ # A worker prefix must not make result.json a valid coverage receipt.
+ return ref
+ ref = normalized
+ provenance = item.get("provenance")
+ inherited_scan_refs = (
+ provenance.get("scanReceiptRefs", []) if isinstance(provenance, dict) else []
+ )
+ if ref.startswith(worker_prefix) or (
+ isinstance(inherited_scan_refs, list) and ref in inherited_scan_refs
+ ):
+ return ref
+ local_ref = Path(f"{directory}/{ref}").as_posix()
+ return local_ref if scan_receipt(local_ref) or not scan_receipt(ref) else ref
+
+ return [qualified_receipt(ref) for ref in refs]
+
+
+def _deferred_candidate_id(
+ row: dict[str, Any],
+ owner: str | None,
+ ambiguous_deferred: set[tuple[str | None, str]],
+) -> str | None:
+ identity = row.get("candidateId") or row.get("id")
+ if not isinstance(identity, str):
+ return None
+ if (owner, identity) in ambiguous_deferred and not any(
+ key in row for key in ("candidateId", "candidate", "finding")
+ ):
+ return None
+ return identity
+
+
+def _generic_surface_updates(
+ sources: list[tuple[str, dict[str, Any], str | None]],
+ closed_deferred: dict[tuple[str | None, str], tuple[tuple[int, int], dict[str, Any], str]],
+ active_deferred: dict[tuple[str | None, str], tuple[tuple[int, int], dict[str, Any], str]],
+ resolved_candidates: dict[tuple[str | None, str], str],
+ reopened_generic: set[tuple[str | None, str]],
+ surface_schema: dict[str, Any],
+ deferred_rows: dict[str, list[Any]],
+ ambiguous_deferred: set[tuple[str | None, str]],
+ record_source: Any,
+ record_order: Any,
+ surface_receipts: Any,
+ *,
+ canonical_rows: set[int] | None = None,
+ coverage_sources: dict[int, tuple[dict[str, Any], list[str]]] | None = None,
+ surface_is_retained: Any = None,
+ eligible_receipt_sources: set[str] | None = None,
+ receipt_root: Path | None = None,
+ receipt_warnings: list[str] | None = None,
+ fallback_surfaces: list[dict[str, Any]] | None = None,
+) -> tuple[set[int], list[tuple[str, str | None, dict[str, Any], dict[str, Any]]]]:
+ if not closed_deferred and not reopened_generic and receipt_root is None:
+ return set(), []
+
+ def receipt_error(ref: Any) -> ContractError | None:
+ if receipt_root is None or not isinstance(ref, str):
+ return ContractError("Saved coverage receipt is unavailable.")
+ try:
+ descriptor = open_scan_local_file_descriptor(
+ receipt_root, ref, "Saved coverage receipt"
+ )
+ os.close(descriptor)
+ except ContractError as exc:
+ return exc
+ return None
+
+ def record_key(field: str, owner: str | None, row: dict[str, Any]) -> tuple[str | None, Any]:
+ source_owner, original = record_source(field, owner, row)
+ identity = original.get("id")
+ return source_owner, identity if isinstance(identity, str) else None
+
+ def linked(row: dict[str, Any], identity: str | None, owner: str | None) -> bool:
+ _, row = record_source("deferred", owner, row)
+ surface_ids = row.get("surfaceIds", [])
+ return identity is not None and (
+ row.get("id") == identity or (isinstance(surface_ids, list) and identity in surface_ids)
+ )
+
+ saved_surfaces: dict[tuple[str | None, str], list[tuple[str, dict[str, Any]]]] = {}
+ for relative, draft, owner in sources:
+ rows = draft["coverage"].get("surfaces", [])
+ for row in rows if isinstance(rows, list) else []:
+ if isinstance(row, dict) and isinstance(row.get("id"), str):
+ saved_surfaces.setdefault(record_key("surfaces", owner, row), []).append(
+ (relative, row)
+ )
+ replaced: set[int] = set()
+ updates: list[tuple[str, str | None, dict[str, Any], dict[str, Any]]] = []
+ # Current selections across all surfaces take precedence over historical refs.
+ receipt_paths: dict[str, str] = {}
+ retained_surfaces: dict[tuple[str | None, str], dict[str, Any]] = {}
+ if receipt_root is not None:
+ receipt_surfaces: list[tuple[str | None, dict[str, Any]]] = []
+ for key, matches in saved_surfaces.items():
+ retained = [
+ match for match in matches if surface_is_retained(match[0], key[0], match[1])
+ ]
+ if not retained:
+ continue
+ latest_source, latest_surface = max(
+ retained, key=lambda match: record_order(match[0], key[0], match[1])
+ )
+ retained_surfaces[key] = latest_surface
+ receipt_surfaces.extend(
+ (key[0], row)
+ for relative, row in retained
+ if record_order(relative, key[0], row)
+ == record_order(latest_source, key[0], latest_surface)
+ )
+ for relative, draft, owner in sources:
+ rows = draft["coverage"].get("surfaces", [])
+ for row in rows if isinstance(rows, list) else []:
+ if (
+ isinstance(row, dict)
+ and "id" not in row
+ and surface_is_retained(relative, owner, row)
+ ):
+ receipt_surfaces.append((owner, row))
+ receipt_surfaces.extend((None, row) for row in fallback_surfaces or [])
+ for owner, latest_surface in receipt_surfaces:
+ # Recovery repairs disposition and receipts before validating row
+ # metadata. A row discarded there cannot reserve a receipt spelling.
+ metadata = _project_worker_coverage_row(latest_surface, owner, "surfaces")
+ metadata.setdefault("id", _saved_coverage_id(metadata))
+ metadata.update(disposition="needs_follow_up", receiptRefs=[])
+ try:
+ _require_str(metadata, "id", "coverage.surfaces")
+ _validate_schema_node(metadata, surface_schema, "coverage.surfaces")
+ except ContractError:
+ continue
+ refs = latest_surface.get("receiptRefs", [])
+ for ref in refs if isinstance(refs, list) else []:
+ if receipt_error(ref) is None:
+ normalized = _require_portable_relative_path(ref, "Saved coverage receipt")
+ receipt_paths.setdefault(normalized.lower(), normalized)
+ for relative, draft, owner in sources:
+ current = draft["coverage"].get("surfaces", [])
+ for surface in current if isinstance(current, list) else []:
+ if not isinstance(surface, dict):
+ continue
+ surface_owner, identity = record_key("surfaces", owner, surface)
+ closed_ids = {
+ identity
+ for (saved_owner, identity), (_, _, source) in closed_deferred.items()
+ if saved_owner == surface_owner and source == relative
+ }
+ reopened_ids = {
+ identity
+ for (saved_owner, identity), (_, _, source) in active_deferred.items()
+ if saved_owner == surface_owner
+ and source == relative
+ and (owner, identity) in reopened_generic
+ }
+ reopening = surface.get("disposition") == "needs_follow_up"
+ work_ids = reopened_ids if reopening else closed_ids
+ repair_only = not work_ids
+ if repair_only and receipt_root is None:
+ continue
+ if not isinstance(identity, str):
+ continue
+ matches = saved_surfaces[(surface_owner, identity)]
+ # Receipt repair alone cannot close or reopen a saved surface.
+ if repair_only and any(
+ row.get("disposition") != surface.get("disposition") for _, row in matches
+ ):
+ continue
+ # Older writers could assign one ID to distinct surfaces in a draft.
+ # A closure cannot identify which of those observations it replaces.
+ by_source = dict(matches)
+ if any(row != by_source[saved_path] for saved_path, row in matches):
+ continue
+ if any(
+ "candidateId" in row or "candidate" in row or "finding" in row for _, row in matches
+ ):
+ continue
+ if any(
+ row is not surface
+ and record_order(saved_path, surface_owner, row)
+ >= record_order(relative, surface_owner, surface)
+ and row.get("disposition") != surface.get("disposition")
+ for saved_path, row in matches
+ ):
+ continue
+
+ if not reopening and any(
+ saved_owner == surface_owner
+ and (
+ not isinstance(row.get("id") or row.get("candidateId"), str)
+ or (
+ isinstance(row.get("id"), str)
+ and (owner, row["id"]) in ambiguous_deferred
+ and not any(key in row for key in ("candidateId", "candidate", "finding"))
+ )
+ )
+ and linked(row, identity, saved_owner)
+ for saved_path, _, saved_owner in sources
+ for row in deferred_rows[saved_path]
+ if isinstance(row, dict)
+ ):
+ continue
+ if not reopening and any(
+ saved_owner == surface_owner
+ and (owner, deferred_id) not in closed_deferred
+ and (
+ (candidate_id := _deferred_candidate_id(row, owner, ambiguous_deferred)) is None
+ or (owner, candidate_id) not in resolved_candidates
+ )
+ and linked(row, identity, saved_owner)
+ for (saved_owner, deferred_id), (_, row, _) in active_deferred.items()
+ ):
+ continue
+ # An accepted checkpoint can update a saved surface by ID without
+ # optional surfaceIds links on its generic task.
+ receipt_matches = [
+ (saved_path, row)
+ for saved_path, row in matches
+ if receipt_root is None
+ or saved_path in eligible_receipt_sources
+ or surface_is_retained(saved_path, owner, row)
+ ]
+ if not receipt_matches:
+ continue
+ latest_relative, latest_surface = max(
+ receipt_matches, key=lambda match: record_order(match[0], surface_owner, match[1])
+ )
+ target = (
+ retained_surfaces.get((surface_owner, identity)) if repair_only else latest_surface
+ )
+ if target is None:
+ continue
+ update = copy.deepcopy(target)
+ # Historical evidence can repair a retained row without promoting
+ # an unselected observation's label, notes, or other metadata.
+ refs = copy.deepcopy(surface_receipts(latest_relative, latest_surface))
+ update["receiptRefs"] = refs
+ replaced_receipt = False
+ replacement_paths = dict(receipt_paths)
+ if isinstance(refs, list):
+ repaired = (
+ not reopening and bool(refs) and all(receipt_error(ref) is None for ref in refs)
+ )
+ if repaired:
+ for ref in refs:
+ normalized = _require_portable_relative_path(ref, "Saved coverage receipt")
+ key = normalized.lower()
+ if key in replacement_paths and replacement_paths[key] != normalized:
+ repaired = False
+ break
+ replacement_paths[key] = normalized
+ for saved_relative, row in receipt_matches:
+ previous_refs = surface_receipts(saved_relative, row)
+ for ref in previous_refs if isinstance(previous_refs, list) else []:
+ if ref not in refs:
+ failure = receipt_error(ref) if repaired else None
+ replaceable = failure is not None and isinstance(
+ failure.__cause__, FileNotFoundError
+ )
+ if repaired and isinstance(ref, str):
+ try:
+ normalized = _require_portable_relative_path(
+ ref, "Saved coverage receipt"
+ )
+ except ContractError as error:
+ failure = error
+ replaceable = True
+ else:
+ key = normalized.lower()
+ previous = replacement_paths.get(key)
+ if previous is not None and previous != normalized:
+ failure = ContractError(
+ f"Saved coverage receipt: duplicate artifact path: {previous} and {normalized}"
+ )
+ replaceable = True
+ elif not replaceable:
+ replacement_paths[key] = normalized
+ if replaceable:
+ replaced_receipt = True
+ if receipt_warnings is not None:
+ receipt_warnings.append(
+ f"Replaced invalid saved coverage receipt {ref}: {failure}"
+ )
+ continue
+ refs.append(ref)
+ if repair_only and not replaced_receipt:
+ continue
+ try:
+ _validate_schema_node(update, surface_schema, "coverage.surfaces")
+ except ContractError:
+ continue
+ replaced_sources = {
+ id(row)
+ for _, row in matches
+ if reopening
+ or row.get("disposition") in {"needs_follow_up", surface.get("disposition")}
+ }
+ replaced.update(replaced_sources)
+ entry = (latest_relative, surface_owner, latest_surface, update)
+ if entry not in updates:
+ receipt_paths.update(replacement_paths)
+ updates.append(entry)
+ retained = next(
+ entry[3]
+ for entry in updates
+ if entry[:3] == (latest_relative, surface_owner, latest_surface)
+ and entry[3] == update
+ )
+ if coverage_sources is not None:
+ from workbench.retained_coverage import _inherit_coverage_sources
+
+ _inherit_coverage_sources(
+ retained,
+ [latest_surface, *(row for _, row in matches if id(row) in replaced_sources)],
+ coverage_sources,
+ )
+ if canonical_rows is not None and id(latest_surface) in canonical_rows:
+ canonical_rows.add(id(retained))
+ return replaced, updates
diff --git a/plugins/codex-security/scripts/workbench_saved_results.py b/plugins/codex-security/scripts/workbench_saved_results.py
index 75e6d5900..8ec15dfdb 100644
--- a/plugins/codex-security/scripts/workbench_saved_results.py
+++ b/plugins/codex-security/scripts/workbench_saved_results.py
@@ -44,12 +44,6 @@ from workbench.retained_coverage import (
from workbench.retained_coverage import (
_coverage_rows as _coverage_rows,
)
-from workbench.retained_coverage import (
- _deferred_candidate_id as _deferred_candidate_id,
-)
-from workbench.retained_coverage import (
- _deferred_rows as _deferred_rows,
-)
from workbench.retained_coverage import (
_ensure_finding_identities as _ensure_finding_identities,
)
@@ -65,9 +59,6 @@ from workbench.retained_coverage import (
from workbench.retained_coverage import (
_finding_key as _finding_key,
)
-from workbench.retained_coverage import (
- _generic_surface_updates as _generic_surface_updates,
-)
from workbench.retained_coverage import (
_has_candidate_metadata as _has_candidate_metadata,
)
@@ -77,12 +68,6 @@ from workbench.retained_coverage import (
from workbench.retained_coverage import (
_inherit_coverage_sources as _inherit_coverage_sources,
)
-from workbench.retained_coverage import (
- _merge_tied_parent_observations as _merge_tied_parent_observations,
-)
-from workbench.retained_coverage import (
- _resolved_deferred_rows as _resolved_deferred_rows,
-)
from workbench.retained_coverage import (
_retained_findings as _retained_findings,
)
@@ -93,6 +78,15 @@ from workbench.retained_coverage import (
_worker_candidate_key as _worker_candidate_key,
)
from workbench.retained_coverage import merge_saved_results as merge_saved_results
+from workbench.saved_coverage import (
+ _deferred_rows as _deferred_rows,
+)
+from workbench.saved_coverage import (
+ _merge_tied_parent_observations as _merge_tied_parent_observations,
+)
+from workbench.saved_coverage import (
+ _resolved_deferred_rows as _resolved_deferred_rows,
+)
from workbench.saved_coverage import (
_saved_coverage_id as _saved_coverage_id,
)
@@ -133,6 +127,12 @@ from workbench.saved_sources import (
_worker_outputs as _worker_outputs,
)
from workbench_constants import PHASES
+from workbench_coverage_receipts import (
+ _deferred_candidate_id as _deferred_candidate_id,
+)
+from workbench_coverage_receipts import (
+ _generic_surface_updates as _generic_surface_updates,
+)
_PUBLICATION_FOLLOW_UP_WARNING = (
"Saved scan evidence remains on disk; result publication needs follow-up:"
diff --git a/plugins/codex-security/scripts/workbench_validation.py b/plugins/codex-security/scripts/workbench_validation.py
index a03810d02..8cdb31533 100644
--- a/plugins/codex-security/scripts/workbench_validation.py
+++ b/plugins/codex-security/scripts/workbench_validation.py
@@ -25,6 +25,15 @@ from workbench_scan_usage import _timestamp as usage_timestamp
reject_non_finite_json = finalizer._reject_non_finite_json
+def javascript_utf8(value: str) -> bytes:
+ """Match Node's UTF-8 encoding of UTF-16 strings without changing stored text."""
+ return (
+ value.encode("utf-16-le", errors="surrogatepass")
+ .decode("utf-16-le", errors="replace")
+ .encode("utf-8")
+ )
+
+
def timestamp_key(value: str) -> datetime:
return finalizer.parse_timestamp(value)
diff --git a/plugins/codex-security/tests/test_cli_dry_runs.py b/plugins/codex-security/tests/test_cli_dry_runs.py
index 3c1c451d6..27767c7e1 100644
--- a/plugins/codex-security/tests/test_cli_dry_runs.py
+++ b/plugins/codex-security/tests/test_cli_dry_runs.py
@@ -22,6 +22,8 @@ def run_script(name: str, *args: str, isolated: bool = False) -> subprocess.Comp
def test_all_scripts_support_help() -> None:
for script in sorted(SCRIPT_DIR.glob("*.py")):
+ if script.name == "workbench_coverage_receipts.py":
+ continue
result = run_script(script.name, "--help")
assert result.returncode == 0, f"{script.name}: {result.stderr}"
assert "usage:" in result.stdout.lower(), script.name
diff --git a/plugins/codex-security/tests/test_deep_coverage_recovery.py b/plugins/codex-security/tests/test_deep_coverage_recovery.py
index 1eb8982bf..eda32ba35 100644
--- a/plugins/codex-security/tests/test_deep_coverage_recovery.py
+++ b/plugins/codex-security/tests/test_deep_coverage_recovery.py
@@ -91,6 +91,58 @@ def test_worker_projection_retains_independent_links_receipts_and_frozen_replay(
assert all(p.read_bytes() == contents for p, contents in originals.items())
+def test_accepted_attempt_reconciles_legacy_parent_worker_projection(tmp_path):
+ surface = {
+ "id": "api",
+ "label": "Worker review",
+ "disposition": "no_issue_found",
+ "receiptRefs": ["artifacts/review.md"],
+ }
+ task = {"id": "review", "reason": "Review remains.", "surfaceIds": ["api"]}
+ source = worker(
+ tmp_path,
+ "first",
+ saved_draft("coverage-scan", complete=True, surfaces=[surface], deferred=[task]),
+ )
+ source.update(
+ status="succeeded", result_manifest_path=str(Path(source["artifact_dir"]) / "result.json")
+ )
+ write_saved_parent(
+ tmp_path,
+ saved_draft(
+ "coverage-scan",
+ complete=True,
+ surfaces=[
+ {
+ **surface,
+ "id": "first/api",
+ "workerId": "first",
+ "receiptRefs": [
+ "artifacts/deep_discovery/workers/first/output/artifacts/review.md"
+ ],
+ }
+ ],
+ deferred=[
+ {**task, "id": "first/review", "workerId": "first", "surfaceIds": ["first/api"]}
+ ],
+ ),
+ 200,
+ )
+ originals = {path: path.read_bytes() for path in tmp_path.rglob("*.json")}
+ first = recover(tmp_path, [source], project=True)
+ replay = recover(tmp_path, [source], project=True, frozen=first[0]["scan"]["preservedSources"])
+ for documents in (first, replay):
+ surfaces = documents[2]["surfaces"]
+ tasks = [row for row in documents[2]["deferred"] if row["id"] != "scan-stopped"]
+ assert len(surfaces) == len(tasks) == 1
+ assert surfaces[0]["provenance"] == {"workerId": "first", "attempt": 1, "sourceId": "api"}
+ assert tasks[0]["provenance"] == {"workerId": "first", "attempt": 1, "sourceId": "review"}
+ assert tasks[0]["surfaceIds"] == [surfaces[0]["id"]]
+ assert (tmp_path / surfaces[0]["receiptRefs"][0]).read_text() == "first source review\n"
+ assert replay[2] == first[2]
+ assert all(path.read_bytes() == contents for path, contents in originals.items())
+
+
def test_partial_parent_without_arrays_retains_closed_worker_surface_and_frozen_replay(tmp_path):
surface = {"id": "api", "label": "API", "disposition": "needs_follow_up", "receiptRefs": []}
pending = saved_draft(
@@ -1025,7 +1077,6 @@ def test_current_receipt_spelling_precedes_historical_receipts(
warnings = []
_, updates = saved._generic_surface_updates(
sources,
- {"old": (0, 0), "current": (1, 0)},
{("first", "review"): ((1, 0), {"id": "review", "reason": "Review completed."}, "current")},
{},
{},
@@ -1033,14 +1084,15 @@ def test_current_receipt_spelling_precedes_historical_receipts(
schema,
{"old": [task], "current": []},
set(),
- canonical_rows=set(),
- coverage_sources={},
+ lambda _field, owner, row: (owner, row),
+ lambda relative, _owner, _row: {"old": (0, 0), "current": (1, 0)}[relative],
+ lambda _relative, row: row.get("receiptRefs", []),
surface_is_retained=lambda _relative, _owner, _row: True,
eligible_receipt_sources={"old", "current"},
receipt_root=tmp_path,
receipt_warnings=warnings,
)
- receipts = {row["id"]: row["receiptRefs"] for _, row, _ in updates}
+ receipts = {row["id"]: row["receiptRefs"] for _, _, _, row in updates}
assert receipts == {"reader": [prefix + "new-review.md"], "writer": [prefix + "Review.md"]}
assert any("duplicate artifact path" in warning for warning in warnings)
scan = {
diff --git a/plugins/codex-security/tests/test_deep_deadline_recovery_edges.py b/plugins/codex-security/tests/test_deep_deadline_recovery_edges.py
index 654c51bd1..1eedf264f 100644
--- a/plugins/codex-security/tests/test_deep_deadline_recovery_edges.py
+++ b/plugins/codex-security/tests/test_deep_deadline_recovery_edges.py
@@ -524,8 +524,10 @@ def test_failed_head_snapshot_does_not_interrupt_publication_rollback(
@pytest.mark.parametrize("directory", ["checkpoints", "checkpoint-heads", "source-order"])
def test_deadline_rollback_preserves_preexisting_dangling_recovery_entry(
- tmp_path: Path, workbench_api, directory: str
+ tmp_path: Path, workbench_api, monkeypatch, directory: str
):
+ from workbench import retained_coverage, saved_sources
+
scan = deadline_fixture(tmp_path)
canonical_parent(scan, saved_draft(scan.scan_id, findings=[scan.finding]))
saved = workbench_api["saved_results"]
@@ -562,6 +564,16 @@ def test_deadline_rollback_preserves_preexisting_dangling_recovery_entry(
raise
with pytest.raises(saved.ContractError) as rejected:
saved.write_scan_local_bytes(scan.directory, relative.as_posix(), b"{}")
+ write = saved.write_scan_local_bytes
+ attempted_writes = set()
+
+ def record_write(directory, relative, contents):
+ attempted_writes.add(relative)
+ write(directory, relative, contents)
+
+ monkeypatch.setattr(saved, "write_scan_local_bytes", record_write)
+ monkeypatch.setattr(saved_sources, "write_scan_local_bytes", record_write)
+ monkeypatch.setattr(retained_coverage, "write_scan_local_bytes", record_write)
try:
publication = saved.preserve_deep_deadline_draft(
SimpleNamespace(**workbench_api), connection, record
@@ -574,7 +586,10 @@ def test_deadline_rollback_preserves_preexisting_dangling_recovery_entry(
"SELECT completion_warnings_json FROM scans WHERE id = ?", (scan.scan_id,)
).fetchone()[0]
)
- assert any(str(rejected.value) in warning for warning in warnings)
+ # Existing checkpoint names participate in the parent snapshot digest,
+ # so this entry can remain untouched while a new checkpoint is written.
+ if relative.as_posix() in attempted_writes:
+ assert any(str(rejected.value) in warning for warning in warnings)
connection.rollback()
saved._restore_published_outputs(*publication)
assert target.is_symlink()
diff --git a/plugins/codex-security/tests/test_partial_coverage_projection.py b/plugins/codex-security/tests/test_partial_coverage_projection.py
new file mode 100644
index 000000000..bb4cad333
--- /dev/null
+++ b/plugins/codex-security/tests/test_partial_coverage_projection.py
@@ -0,0 +1,2143 @@
+from __future__ import annotations
+
+import copy
+import hashlib
+import json
+import os
+import uuid
+from argparse import Namespace
+
+import pytest
+from test_deep_scan_successful_publication import add_worker
+from test_deep_scan_successful_publication import publication_scan as publication_scan
+from workbench_test_support import write_checkpoint
+
+
[email protected]("attempt", [1, 2])
[email protected]("has_head", [False, True])
[email protected]("newer_parent", [False, True])
[email protected]("retry_publication", [False, True])
+def test_parent_candidate_outcome_compares_headless_worker_chronology(
+ workbench_api,
+ workbench_db,
+ publication_scan,
+ monkeypatch,
+ attempt,
+ has_head,
+ newer_parent,
+ retry_publication,
+):
+ scan = publication_scan()
+ result = add_worker(workbench_db, scan)
+ worker_id = result.parent.name
+ with workbench_db:
+ workbench_db.execute(
+ "UPDATE deep_scan_workers SET attempt = ? WHERE id = ?", (attempt, worker_id)
+ )
+ candidate = "pending-candidate"
+ surface = {
+ "id": "worker-surface",
+ "candidateId": candidate,
+ "label": "Worker follow-up",
+ "disposition": "needs_follow_up",
+ "receiptRefs": [],
+ }
+ task = {"id": "worker-task", "candidateId": candidate, "reason": "Validate worker evidence."}
+ draft = {
+ "scanId": scan.scan_id,
+ "complete": True,
+ "findings": [],
+ "coverage": {
+ **scan.coverage,
+ "completeness": "partial",
+ "surfaces": [surface],
+ "deferred": [task],
+ },
+ }
+ result.write_text(json.dumps(draft))
+ os.utime(result, ns=(300, 300))
+ originals = {result: result.read_bytes()}
+ if has_head:
+ checkpoint = write_checkpoint(result.parent / "checkpoints", draft)
+ os.utime(checkpoint, ns=(300, 300))
+ head = result.parent / "checkpoint-head.json"
+ head.write_text(json.dumps({"checkpoint": checkpoint.name}))
+ os.utime(head, ns=(300, 300))
+ originals.update({path: path.read_bytes() for path in (checkpoint, head)})
+ (scan.scan_dir / "findings.json").write_text(json.dumps({"findings": []}))
+ parent = scan.scan_dir / "coverage.json"
+ parent.write_text(
+ json.dumps(
+ {
+ **scan.coverage,
+ "surfaces": [
+ {
+ **surface,
+ "id": "parent-rejection",
+ "label": "Parent reviewed candidate",
+ "disposition": "rejected",
+ "provenance": {
+ "workerId": worker_id,
+ "attempt": attempt,
+ "candidateId": candidate,
+ "sourceId": surface["id"],
+ },
+ }
+ ],
+ "reviews": [{"workerId": worker_id, "attempt": attempt, "completeness": "partial"}],
+ }
+ )
+ )
+ modified = 400 if newer_parent else 200
+ os.utime(parent, ns=(modified, modified))
+ with monkeypatch.context() as interrupted:
+ if retry_publication:
+
+ def fail_publication(*args, **kwargs):
+ raise OSError("Synthetic publication interruption.")
+
+ interrupted.setattr(
+ workbench_api["saved_results"],
+ "_write_prepared_scan_finalization",
+ fail_publication,
+ )
+ workbench_api["saved_results"].fail_scan(
+ workbench_api["_WORKBENCH_DB_CONTEXT"],
+ workbench_db,
+ Namespace(scan_id=scan.scan_id, claim_token=None, cost_json=None, message="Stopped."),
+ )
+ workbench_api["saved_results"].recover_scan_results(
+ workbench_api["_WORKBENCH_DB_CONTEXT"], workbench_db, Namespace(scan_id=scan.scan_id)
+ )
+ coverage = json.loads(parent.read_text())
+ pending = [row for row in coverage["deferred"] if row.get("reason") == task["reason"]]
+ assert len(pending) == int(not newer_parent)
+ assert any(
+ row.get("label") == surface["label"] and row.get("disposition") == "needs_follow_up"
+ for row in coverage["surfaces"]
+ ) is (not newer_parent)
+ assert all(path.read_bytes() == contents for path, contents in originals.items())
+
+
+ "parent_surfaces", ["missing", "projected", "renamed", "second-only", "no-parent", "edited"]
+)
[email protected]("tagged_surface", [False, True])
[email protected]("merge_state", ["buffered", "merging", "merged"])
+def test_missing_deferred_projection_links_first_duplicate_surface(
+ workbench_api, workbench_db, publication_scan, parent_surfaces, merge_state, tagged_surface
+):
+ scan = publication_scan()
+ result = add_worker(workbench_db, scan)
+ worker_id = result.parent.name
+ with workbench_db:
+ workbench_db.execute(
+ "UPDATE deep_scan_workers SET merge_state = ? WHERE id = ?",
+ (merge_state, worker_id),
+ )
+ surfaces = [
+ {
+ "id": "shared-surface",
+ "label": label,
+ "disposition": disposition,
+ "receiptRefs": [],
+ }
+ for label, disposition in (
+ ("Archive route", "needs_follow_up"),
+ ("Archive settings", "no_issue_found"),
+ )
+ ]
+ if tagged_surface:
+ surfaces[0]["candidateId"] = "source-candidate"
+ deferred = {
+ "id": "pending",
+ "reason": "Verify entry boundaries.",
+ "surfaceIds": ["shared-surface"],
+ }
+ result.write_text(
+ json.dumps(
+ {
+ "scanId": scan.scan_id,
+ "complete": True,
+ "findings": [],
+ "coverage": {
+ **scan.coverage,
+ "completeness": "partial",
+ "surfaces": surfaces,
+ "deferred": [deferred],
+ },
+ }
+ )
+ )
+ original = result.read_bytes()
+ projected = [
+ {
+ **surface,
+ "id": f"{worker_id}-attempt-1-surface-{index}",
+ "provenance": {"workerId": worker_id, "attempt": 1, "sourceId": surface["id"]},
+ }
+ for index, surface in enumerate(surfaces, 1)
+ ]
+ if tagged_surface:
+ projected[0]["provenance"]["candidateId"] = surfaces[0]["candidateId"]
+ projected[0]["candidateId"] = (
+ f"{worker_id}-attempt-1-candidate-"
+ + hashlib.sha256(surfaces[0]["candidateId"].encode()).hexdigest()
+ )
+ if parent_surfaces == "renamed":
+ projected[0]["id"] = "canonical-first-surface"
+ if parent_surfaces == "edited":
+ projected[0]["label"] = "Parent requested a different review."
+ (scan.scan_dir / "coverage.json").write_text(
+ json.dumps(
+ {
+ **scan.coverage,
+ "completeness": "partial",
+ "reviews": [{"workerId": worker_id, "attempt": 1, "completeness": "partial"}],
+ "deferred": [
+ {
+ "id": f"{worker_id}-attempt-1-deferred-1",
+ "reason": "Parent requested different proof.",
+ "surfaceIds": [projected[0]["id"]],
+ }
+ ]
+ if parent_surfaces == "edited"
+ else [],
+ "surfaces": []
+ if parent_surfaces == "missing"
+ else projected[1:]
+ if parent_surfaces == "second-only"
+ else projected,
+ }
+ )
+ )
+ if parent_surfaces == "no-parent":
+ for name in ("scan-manifest.json", "findings.json", "coverage.json"):
+ (scan.scan_dir / name).unlink()
+ if tagged_surface:
+ earlier = copy.deepcopy(json.loads(result.read_text()))
+ earlier["coverage"]["surfaces"][0]["disposition"] = "no_issue_found"
+ earlier["coverage"]["deferred"] = []
+ earlier["coverage"]["resolvedDeferred"] = [
+ {"id": deferred["id"], "reason": "Earlier source follow-up completed."}
+ ]
+ checkpoint = write_checkpoint(result.parent / "checkpoints", earlier)
+ os.utime(checkpoint, ns=(100, 100))
+ os.utime(result, ns=(300, 300))
+ if parent_surfaces != "no-parent":
+ os.utime(scan.scan_dir / "coverage.json", ns=(200, 200))
+ workbench_api["saved_results"].fail_scan(
+ workbench_api["_WORKBENCH_DB_CONTEXT"],
+ workbench_db,
+ Namespace(scan_id=scan.scan_id, claim_token=None, cost_json=None, message="Stopped."),
+ )
+ recovered = workbench_api["saved_results"].recover_scan_results(
+ workbench_api["_WORKBENCH_DB_CONTEXT"], workbench_db, Namespace(scan_id=scan.scan_id)
+ )["scan"]
+ assert recovered["resultsRecoveryNeeded"] is False
+ coverage_path = scan.scan_dir / "coverage.json"
+ coverage = json.loads(coverage_path.read_text())
+ pending = [item for item in coverage["deferred"] if item.get("reason") == deferred["reason"]]
+ assert len(pending) == 1
+ if parent_surfaces == "edited":
+ worker_surface = next(
+ surface for surface in coverage["surfaces"] if surface["label"] == surfaces[0]["label"]
+ )
+ assert worker_surface["id"] != projected[0]["id"]
+ assert pending[0]["surfaceIds"] == [worker_surface["id"]]
+ parent_task = next(
+ task
+ for task in coverage["deferred"]
+ if task["reason"] == "Parent requested different proof."
+ )
+ assert parent_task["surfaceIds"] == [projected[0]["id"]]
+ assert len(coverage["surfaces"]) == len(projected) + 1
+ else:
+ assert pending[0]["surfaceIds"] == [projected[0]["id"]]
+ assert len(coverage["surfaces"]) == len(projected)
+ assert all(surface in coverage["surfaces"] for surface in projected)
+ assert result.read_bytes() == original
+ published = coverage_path.read_bytes()
+ workbench_api["saved_results"].recover_scan_results(
+ workbench_api["_WORKBENCH_DB_CONTEXT"], workbench_db, Namespace(scan_id=scan.scan_id)
+ )
+ assert coverage_path.read_bytes() == published
+ late = write_checkpoint(
+ result.parent / "checkpoints",
+ {
+ **json.loads(result.read_text()),
+ "complete": False,
+ "coverage": {**scan.coverage, "openQuestions": ["Late worker checkpoint."]},
+ },
+ )
+ original_late = late.read_bytes()
+ workbench_api["saved_results"].recover_scan_results(
+ workbench_api["_WORKBENCH_DB_CONTEXT"], workbench_db, Namespace(scan_id=scan.scan_id)
+ )
+ assert coverage_path.read_bytes() == published
+ assert result.read_bytes() == original
+ assert late.read_bytes() == original_late
+
+
[email protected]("parent_review", [False, True])
+def test_missing_accepted_result_does_not_project_an_archived_attempt(
+ workbench_api, workbench_db, publication_scan, parent_review
+):
+ scan = publication_scan()
+ result = add_worker(workbench_db, scan)
+ worker_id = result.parent.name
+ result = scan.scan_dir / "artifacts" / worker_id / "output" / "result.json"
+ result.parent.mkdir(parents=True)
+ with workbench_db:
+ workbench_db.execute(
+ "UPDATE deep_scan_workers SET attempt = 2, artifact_dir = ?, result_manifest_path = ? WHERE id = ?",
+ (str(result.parent), str(result), worker_id),
+ )
+ archive = result.parent.parent / "attempts" / "attempt-01"
+ receipt = archive / "artifacts" / "prior.txt"
+ receipt.parent.mkdir(parents=True)
+ receipt.write_text("Prior attempt receipt.\n")
+ checkpoint = write_checkpoint(
+ archive / "checkpoints",
+ {
+ "scanId": scan.scan_id,
+ "complete": False,
+ "findings": [],
+ "coverage": {
+ **scan.coverage,
+ "completeness": "partial",
+ "surfaces": [
+ {
+ "id": "prior-surface",
+ "label": "Prior unfinished review",
+ "disposition": "needs_follow_up",
+ "receiptRefs": ["artifacts/prior.txt"],
+ }
+ ],
+ "deferred": [{"reason": "Prior proof gap.", "surfaceIds": ["prior-surface"]}],
+ },
+ },
+ )
+ original = checkpoint.read_bytes()
+ if parent_review:
+ (scan.scan_dir / "coverage.json").write_text(
+ json.dumps(
+ {
+ **scan.coverage,
+ "reviews": [{"workerId": worker_id, "attempt": 2, "completeness": "complete"}],
+ }
+ )
+ )
+ else:
+ for name in ("scan-manifest.json", "findings.json", "coverage.json"):
+ (scan.scan_dir / name).unlink()
+ workbench_api["saved_results"].fail_scan(
+ workbench_api["_WORKBENCH_DB_CONTEXT"],
+ workbench_db,
+ Namespace(scan_id=scan.scan_id, claim_token=None, cost_json=None, message="Stopped."),
+ )
+ workbench_api["saved_results"].recover_scan_results(
+ workbench_api["_WORKBENCH_DB_CONTEXT"], workbench_db, Namespace(scan_id=scan.scan_id)
+ )
+ coverage = json.loads((scan.scan_dir / "coverage.json").read_text())
+ surface = next(
+ item for item in coverage["surfaces"] if item["label"] == "Prior unfinished review"
+ )
+ assert surface["id"] == f"{worker_id}/prior-surface"
+ assert surface["workerId"] == worker_id
+ assert "provenance" not in surface
+ assert surface["receiptRefs"] == [receipt.relative_to(scan.scan_dir).as_posix()]
+ assert len(coverage.get("reviews", [])) == int(parent_review)
+ pending = next(
+ item for item in coverage["deferred"] if item.get("reason") == "Prior proof gap."
+ )
+ assert pending["surfaceIds"] == [surface["id"]]
+ assert checkpoint.read_bytes() == original
+
+
[email protected]("failed_first", [False, True])
+def test_failed_worker_reviews_do_not_replace_accepted_reviews(
+ workbench_api, workbench_db, publication_scan, failed_first
+):
+ scan = publication_scan()
+ # These fixture rows share created_at; recovery reads them in worker-ID order.
+ first, second = sorted([add_worker(workbench_db, scan), add_worker(workbench_db, scan)])
+ failed, accepted = (first, second) if failed_first else (second, first)
+ with workbench_db:
+ workbench_db.execute(
+ "UPDATE deep_scan_workers SET status = 'failed', merge_state = 'none' WHERE id = ?",
+ (failed.parent.name,),
+ )
+ review = {"workerId": accepted.parent.name, "attempt": 1, "completeness": "partial"}
+ originals = {}
+ drafts = {}
+ for label, result in (("accepted", accepted), ("failed", failed)):
+ coverage = {
+ **scan.coverage,
+ "completeness": "partial",
+ "surfaces": [
+ {
+ "id": f"{label}-surface",
+ "label": f"{label} source review",
+ "disposition": "needs_follow_up",
+ "receiptRefs": [],
+ }
+ ],
+ "deferred": [
+ {
+ "id": f"{label}-gap",
+ "reason": f"{label} source needs proof.",
+ "surfaceIds": [f"{label}-surface"],
+ }
+ ],
+ "explicitExclusions": [
+ {
+ "id": f"{label}-exclusion",
+ "pattern": "vendor/",
+ "reason": f"{label} source excludes external dependencies.",
+ }
+ ],
+ "openQuestions": [{"question": f"Which controls apply to the {label} source?"}],
+ }
+ if result == failed:
+ coverage["reviews"] = [{**review, "completeness": "complete"}]
+ draft = {
+ "scanId": scan.scan_id,
+ "complete": result == accepted,
+ "findings": [],
+ "coverage": coverage,
+ }
+ result.write_text(json.dumps(draft))
+ checkpoint = write_checkpoint(result.parent / "checkpoints", draft)
+ originals.update({path: path.read_bytes() for path in (result, checkpoint)})
+ drafts[label] = draft
+ for name in ("scan-manifest.json", "findings.json", "coverage.json"):
+ (scan.scan_dir / name).unlink()
+ workbench_api["saved_results"].fail_scan(
+ workbench_api["_WORKBENCH_DB_CONTEXT"],
+ workbench_db,
+ Namespace(scan_id=scan.scan_id, claim_token=None, cost_json=None, message="Stopped."),
+ )
+ coverage_path = scan.scan_dir / "coverage.json"
+ coverage = json.loads(coverage_path.read_text())
+ assert coverage["reviews"] == [review]
+ for field in ("surfaces", "deferred", "explicitExclusions", "openQuestions"):
+ for item in drafts["failed"]["coverage"][field]:
+ expected = {**item, "workerId": failed.parent.name}
+ if field != "openQuestions":
+ expected["id"] = f"{failed.parent.name}/{item['id']}"
+ if field == "deferred":
+ expected["surfaceIds"] = [
+ f"{failed.parent.name}/{surface_id}" for surface_id in item["surfaceIds"]
+ ]
+ assert expected in coverage[field]
+ assert any(
+ item.get("reason") == "accepted source needs proof." for item in coverage["deferred"]
+ )
+ published = coverage_path.read_bytes()
+ workbench_api["saved_results"].recover_scan_results(
+ workbench_api["_WORKBENCH_DB_CONTEXT"], workbench_db, Namespace(scan_id=scan.scan_id)
+ )
+ assert coverage_path.read_bytes() == published
+ late = write_checkpoint(
+ accepted.parent / "checkpoints",
+ {
+ **drafts["accepted"],
+ "complete": False,
+ "coverage": {**scan.coverage, "openQuestions": ["Late worker checkpoint."]},
+ },
+ )
+ originals[late] = late.read_bytes()
+ recovered = workbench_api["saved_results"].recover_scan_results(
+ workbench_api["_WORKBENCH_DB_CONTEXT"], workbench_db, Namespace(scan_id=scan.scan_id)
+ )["scan"]
+ assert recovered["resultsRecoveryNeeded"] is False
+ assert coverage_path.read_bytes() == published
+ manifest = json.loads((scan.scan_dir / "scan-manifest.json").read_text())
+ assert late.relative_to(scan.scan_dir).as_posix() in manifest["scan"]["preservedSources"]
+ assert all(path.read_bytes() == original for path, original in originals.items())
+
+
[email protected]("worker_count", [1, 2])
[email protected]("missing_parent_surfaces", [False, True])
[email protected]("retained_deferred", [False, True])
[email protected]("idless_surface", [False, True])
+def test_missing_projection_keeps_surface_links_and_independent_reviews(
+ workbench_api,
+ workbench_db,
+ publication_scan,
+ worker_count,
+ missing_parent_surfaces,
+ retained_deferred,
+ idless_surface,
+):
+ scan = publication_scan()
+ surface = {
+ "id": "source-surface",
+ "label": "Source review",
+ "disposition": "needs_follow_up",
+ "receiptRefs": [],
+ }
+ deferred = {
+ "id": "pending-check",
+ "reason": "Validation remains unresolved.",
+ "candidateId": "pending-candidate",
+ "surfaceIds": [surface["id"]],
+ }
+ worker_surfaces = (
+ [{"label": "Background review", "disposition": "reviewed", "receiptRefs": []}]
+ if idless_surface
+ else []
+ ) + [surface]
+ reviews, surfaces, deferred_records, originals = [], [], [], {}
+ for _ in range(worker_count):
+ result = add_worker(workbench_db, scan)
+ worker_id = result.parent.name
+ reviews.append({"workerId": worker_id, "attempt": 1, "completeness": "partial"})
+ if idless_surface:
+ surfaces.append(
+ {
+ **worker_surfaces[0],
+ "id": f"{worker_id}-attempt-1-surface-1",
+ "provenance": {"workerId": worker_id, "attempt": 1},
+ }
+ )
+ surfaces.append(
+ {
+ **surface,
+ "id": f"{worker_id}-attempt-1-surface-{len(worker_surfaces)}",
+ "provenance": {"workerId": worker_id, "attempt": 1, "sourceId": surface["id"]},
+ }
+ )
+ deferred_records.append(
+ {
+ **deferred,
+ "id": f"{worker_id}-attempt-1-deferred-1",
+ "candidateId": f"{worker_id}-attempt-1-candidate-{hashlib.sha256(deferred['candidateId'].encode()).hexdigest()}",
+ "surfaceIds": [surfaces[-1]["id"]],
+ "provenance": {
+ "workerId": worker_id,
+ "attempt": 1,
+ "sourceId": deferred["id"],
+ "candidateId": deferred["candidateId"],
+ },
+ }
+ )
+ result.write_text(
+ json.dumps(
+ {
+ "scanId": scan.scan_id,
+ "complete": True,
+ "findings": [],
+ "coverage": {
+ **scan.coverage,
+ "completeness": "partial",
+ "surfaces": worker_surfaces,
+ "deferred": [deferred],
+ },
+ }
+ )
+ )
+ originals[result] = result.read_bytes()
+ (scan.scan_dir / "coverage.json").write_text(
+ json.dumps(
+ {
+ **scan.coverage,
+ "completeness": "partial",
+ "surfaces": None if missing_parent_surfaces else surfaces,
+ "deferred": deferred_records if retained_deferred else [],
+ "reviews": reviews,
+ }
+ )
+ )
+ saved = workbench_api["saved_results"]
+ context = workbench_api["_WORKBENCH_DB_CONTEXT"]
+ saved.fail_scan(
+ context,
+ workbench_db,
+ Namespace(
+ scan_id=scan.scan_id,
+ claim_token=None,
+ cost_json=None,
+ message="Stopped.",
+ ),
+ )
+ recovered = saved.recover_scan_results(context, workbench_db, Namespace(scan_id=scan.scan_id))[
+ "scan"
+ ]
+ assert recovered["resultsRecoveryNeeded"] is False
+ coverage = json.loads((scan.scan_dir / "coverage.json").read_text())
+ pending = [item for item in coverage["deferred"] if item.get("reason") == deferred["reason"]]
+ assert len(pending) == worker_count
+ by_surface_id = {item["id"]: item for item in coverage["surfaces"]}
+ for item in pending:
+ if not missing_parent_surfaces:
+ linked = by_surface_id[item["surfaceIds"][0]]
+ assert linked["provenance"]["workerId"] == item["provenance"]["workerId"]
+ assert item["provenance"]["attempt"] == 1
+ assert item["provenance"]["sourceId"] == deferred["id"]
+ assert item["provenance"]["candidateId"] == deferred["candidateId"]
+ assert {item["provenance"]["workerId"] for item in pending} == {
+ item["workerId"] for item in reviews
+ }
+ if missing_parent_surfaces:
+ # The existing finalizer repairs malformed collections and reports a warning.
+ assert coverage["surfaces"] == []
+ assert recovered["warnings"]
+ else:
+ assert len(by_surface_id) == worker_count * len(worker_surfaces)
+ assert "completeness" not in coverage
+ assert all(path.read_bytes() == data for path, data in originals.items())
+ published = (scan.scan_dir / "coverage.json").read_bytes()
+ saved.recover_scan_results(context, workbench_db, Namespace(scan_id=scan.scan_id))
+ assert (scan.scan_dir / "coverage.json").read_bytes() == published
+
+
[email protected]("attempt", [1, 2])
[email protected]("retry_publication", [False, True])
+def test_previous_attempt_resolution_does_not_clear_current_gap(
+ workbench_api, workbench_db, publication_scan, attempt, retry_publication, monkeypatch
+):
+ scan = publication_scan()
+ result = add_worker(workbench_db, scan)
+ worker_id = result.parent.name
+ deferred = {"candidateId": "candidate-1", "reason": "Validate the current attempt."}
+ result.write_text(
+ json.dumps(
+ {
+ "scanId": scan.scan_id,
+ "complete": True,
+ "findings": [],
+ "coverage": {**scan.coverage, "completeness": "partial", "deferred": [deferred]},
+ }
+ )
+ )
+ original = result.read_bytes()
+ with workbench_db:
+ workbench_db.execute(
+ "UPDATE deep_scan_workers SET attempt = ? WHERE id = ?", (attempt, worker_id)
+ )
+ parent_path = scan.scan_dir / "coverage.json"
+ parent_path.write_text(
+ json.dumps(
+ {
+ **scan.coverage,
+ "reviews": [{"workerId": worker_id, "attempt": 1, "completeness": "complete"}],
+ "surfaces": [
+ {
+ "id": "resolved",
+ "label": "Earlier disposition",
+ "disposition": "rejected",
+ "candidateId": "projected-candidate",
+ "receiptRefs": [],
+ "provenance": {
+ "workerId": worker_id,
+ "attempt": 1,
+ "candidateId": "candidate-1",
+ },
+ }
+ ],
+ }
+ )
+ )
+ # This parent disposition follows the worker result; tied observations stay pending.
+ observed = result.stat().st_mtime_ns // 1_000_000_000 * 1_000_000_000
+ os.utime(result, ns=(observed, observed))
+ parent_observed = observed + 2_000_000_000
+ os.utime(parent_path, ns=(parent_observed, parent_observed))
+ with monkeypatch.context() as interrupted:
+ if retry_publication:
+
+ def fail_publication(*args, **kwargs):
+ raise OSError("Synthetic publication interruption.")
+
+ interrupted.setattr(
+ workbench_api["saved_results"],
+ "_write_prepared_scan_finalization",
+ fail_publication,
+ )
+ stopped = workbench_api["saved_results"].fail_scan(
+ workbench_api["_WORKBENCH_DB_CONTEXT"],
+ workbench_db,
+ Namespace(scan_id=scan.scan_id, claim_token=None, cost_json=None, message="Stopped."),
+ )["scan"]
+ assert stopped["resultsRecoveryNeeded"] is retry_publication
+ recovered = workbench_api["saved_results"].recover_scan_results(
+ workbench_api["_WORKBENCH_DB_CONTEXT"], workbench_db, Namespace(scan_id=scan.scan_id)
+ )["scan"]
+ assert recovered["resultsRecoveryNeeded"] is False
+ coverage = json.loads((scan.scan_dir / "coverage.json").read_text())
+ pending = [item for item in coverage["deferred"] if item.get("reason") == deferred["reason"]]
+ assert len(pending) == (0 if attempt == 1 else 1)
+ assert result.read_bytes() == original
+
+
[email protected]("selected_checkpoint", [False, True])
+def test_recovery_compares_open_questions_using_canonical_normalization(
+ workbench_api, workbench_db, publication_scan, selected_checkpoint
+):
+ scan = publication_scan()
+ result = add_worker(workbench_db, scan)
+ worker_id = result.parent.name
+ questions = [
+ " Which deployment controls apply? ",
+ {"question": " Which runtime settings apply? ", "followUpPrompt": " \t"},
+ ]
+ result.write_text(
+ json.dumps(
+ {
+ "scanId": scan.scan_id,
+ "complete": True,
+ "findings": [],
+ "coverage": {**scan.coverage, "openQuestions": questions},
+ }
+ )
+ )
+ original = result.read_bytes()
+ if selected_checkpoint:
+ checkpoint = write_checkpoint(result.parent / "checkpoints", json.loads(result.read_text()))
+ os.utime(result, ns=(100, 100))
+ os.utime(checkpoint, ns=(150, 150))
+ (result.parent / "checkpoint-head.json").write_text(
+ json.dumps({"checkpoint": checkpoint.name})
+ )
+ projected = [
+ {"question": question, "provenance": {"workerId": worker_id, "attempt": 1}}
+ for question in ("Which deployment controls apply?", "Which runtime settings apply?")
+ ]
+ (scan.scan_dir / "coverage.json").write_text(
+ json.dumps(
+ {
+ **scan.coverage,
+ "reviews": [{"workerId": worker_id, "attempt": 1, "completeness": "complete"}],
+ "openQuestions": projected,
+ }
+ )
+ )
+ workbench_api["saved_results"].fail_scan(
+ workbench_api["_WORKBENCH_DB_CONTEXT"],
+ workbench_db,
+ Namespace(scan_id=scan.scan_id, claim_token=None, cost_json=None, message="Stopped."),
+ )
+ recovered = workbench_api["saved_results"].recover_scan_results(
+ workbench_api["_WORKBENCH_DB_CONTEXT"], workbench_db, Namespace(scan_id=scan.scan_id)
+ )["scan"]
+ assert recovered["resultsRecoveryNeeded"] is False
+ coverage = json.loads((scan.scan_dir / "coverage.json").read_text())
+ assert coverage["openQuestions"] == projected
+ assert result.read_bytes() == original
+
+
[email protected]("parent_projection", ["raw", "projected", "missing"])
[email protected]("retry_publication", [False, True])
+def test_rejected_retry_preserves_original_finding_history(
+ workbench_api, workbench_db, publication_scan, monkeypatch, parent_projection, retry_publication
+):
+ scan = publication_scan()
+ result = add_worker(workbench_db, scan)
+ worker_id = result.parent.name
+ candidate_id = "candidate-1"
+ finding = copy.deepcopy(scan.findings[0])
+ finding["provenance"]["candidateId"] = candidate_id
+ previous = result.parent / "attempts" / "attempt-01" / "result.json"
+ previous.parent.mkdir(parents=True)
+ previous.write_text(
+ json.dumps(
+ {
+ "scanId": scan.scan_id,
+ "complete": False,
+ "findings": [finding],
+ "coverage": scan.coverage,
+ }
+ )
+ )
+ surface = {
+ "id": "review",
+ "candidateId": candidate_id,
+ "label": "Rejected candidate",
+ "disposition": "rejected",
+ "receiptRefs": [],
+ }
+ result.write_text(
+ json.dumps(
+ {
+ "scanId": scan.scan_id,
+ "complete": True,
+ "findings": [],
+ "coverage": {**scan.coverage, "surfaces": [surface]},
+ }
+ )
+ )
+ with workbench_db:
+ workbench_db.execute("UPDATE deep_scan_workers SET attempt = 2 WHERE id = ?", (worker_id,))
+ other_result = add_worker(workbench_db, scan)
+ other_result.write_text(
+ json.dumps({"scanId": scan.scan_id, "findings": [], "coverage": scan.coverage})
+ )
+ projected_surfaces = [
+ {
+ **surface,
+ "id": f"{owner}-attempt-{attempt}-surface-1",
+ "candidateId": f"{owner}-attempt-{attempt}-candidate-{hashlib.sha256(candidate_id.encode()).hexdigest()}",
+ "provenance": {
+ "workerId": owner,
+ "attempt": attempt,
+ "sourceId": surface["id"],
+ "candidateId": candidate_id,
+ },
+ }
+ for owner, attempt in ((worker_id, 2), (worker_id, 1), (other_result.parent.name, 1))
+ ]
+ (scan.scan_dir / "findings.json").write_text(json.dumps({"findings": []}))
+ (scan.scan_dir / "coverage.json").write_text(
+ json.dumps(
+ {
+ **scan.coverage,
+ "surfaces": projected_surfaces if parent_projection == "projected" else [surface],
+ "reviews": [
+ {"workerId": owner, "attempt": attempt, "completeness": "complete"}
+ for owner, attempt in (
+ (worker_id, 2),
+ (worker_id, 1),
+ (other_result.parent.name, 1),
+ )
+ ]
+ if parent_projection == "projected"
+ else [],
+ }
+ )
+ )
+ if parent_projection == "missing":
+ for name in ("scan-manifest.json", "findings.json", "coverage.json"):
+ (scan.scan_dir / name).unlink()
+ originals = {path: path.read_bytes() for path in (previous, result, other_result)}
+ with monkeypatch.context() as interrupted:
+ if retry_publication:
+
+ def fail_publication(*args, **kwargs):
+ raise OSError("Synthetic publication interruption.")
+
+ interrupted.setattr(
+ workbench_api["saved_results"],
+ "_write_prepared_scan_finalization",
+ fail_publication,
+ )
+ stopped = workbench_api["saved_results"].fail_scan(
+ workbench_api["_WORKBENCH_DB_CONTEXT"],
+ workbench_db,
+ Namespace(scan_id=scan.scan_id, claim_token=None, cost_json=None, message="Stopped."),
+ )["scan"]
+ assert stopped["resultsRecoveryNeeded"] is retry_publication
+ recovered = workbench_api["saved_results"].recover_scan_results(
+ workbench_api["_WORKBENCH_DB_CONTEXT"], workbench_db, Namespace(scan_id=scan.scan_id)
+ )["scan"]
+ assert recovered["resultsRecoveryNeeded"] is False
+ assert recovered["findingCount"] == 0
+ coverage = json.loads((scan.scan_dir / "coverage.json").read_text())
+ for item in coverage["surfaces"]:
+ provenance = item.get("provenance")
+ if provenance is None or (
+ provenance["workerId"] == worker_id and provenance["attempt"] == 2
+ ):
+ assert item["previousFindings"] == [finding]
+ else:
+ assert "previousFindings" not in item
+ assert all(path.read_bytes() == original for path, original in originals.items())
+ published = (scan.scan_dir / "coverage.json").read_bytes()
+ workbench_api["saved_results"].recover_scan_results(
+ workbench_api["_WORKBENCH_DB_CONTEXT"], workbench_db, Namespace(scan_id=scan.scan_id)
+ )
+ assert (scan.scan_dir / "coverage.json").read_bytes() == published
+
+
[email protected]("older_complete", [False, True])
+ "recovery", [None, "retry", "missing", "changed", "incomplete", "late", "fallback-late"]
+)
+def test_stopped_recovery_keeps_current_parent_projection(
+ workbench_api, workbench_db, publication_scan, monkeypatch, older_complete, recovery
+):
+ retry_publication = recovery not in {None, "fallback-late"}
+ scan = publication_scan()
+ result = add_worker(workbench_db, scan)
+ worker_id = result.parent.name
+ deferred = {"candidateId": "candidate-1", "reason": "The current review remains unresolved."}
+ result.write_text(
+ json.dumps(
+ {
+ "scanId": scan.scan_id,
+ "complete": True,
+ "findings": [],
+ "coverage": {**scan.coverage, "completeness": "partial", "deferred": [deferred]},
+ }
+ )
+ )
+ reducer = add_worker(workbench_db, scan)
+ reducer.write_text(json.dumps({"scanId": scan.scan_id, "findings": []}))
+ reducer_checkpoint = write_checkpoint(
+ reducer.parent / "checkpoints", json.loads(reducer.read_text())
+ )
+ with workbench_db:
+ workbench_db.execute(
+ "UPDATE deep_scan_workers SET kind = 'dedup', merge_state = 'none' WHERE id = ?",
+ (reducer.parent.name,),
+ )
+ reviews = [{"workerId": worker_id, "attempt": 1, "completeness": "partial"}]
+ obsolete = write_checkpoint(
+ scan.scan_dir / "checkpoints",
+ {
+ "scanId": scan.scan_id,
+ "complete": older_complete,
+ "previousParentCheckpoints": [],
+ "findings": [],
+ "coverage": {
+ **scan.coverage,
+ "reviews": reviews,
+ "openQuestions": ["This question was answered by the final parent draft."],
+ "surfaces": [
+ {
+ "id": "old-disposition",
+ "label": "Earlier disposition",
+ "disposition": "rejected",
+ "candidateId": "projected-candidate",
+ "receiptRefs": [],
+ "provenance": {
+ "workerId": worker_id,
+ "attempt": 1,
+ "candidateId": "candidate-1",
+ },
+ }
+ ],
+ },
+ },
+ )
+ question = {
+ "question": "Which deployment control remains unverified?",
+ "provenance": {"workerId": worker_id, "attempt": 1},
+ }
+ (scan.scan_dir / "coverage.json").write_text(
+ json.dumps(
+ {
+ **scan.coverage,
+ "completeness": "partial",
+ "reviews": reviews,
+ "openQuestions": [question],
+ }
+ )
+ )
+ if recovery == "incomplete":
+ manifest_path = scan.scan_dir / "scan-manifest.json"
+ manifest = json.loads(manifest_path.read_text())
+ manifest["scan"]["complete"] = False
+ manifest_path.write_text(json.dumps(manifest))
+ original_sources = {
+ path: path.read_bytes() for path in (result, reducer, reducer_checkpoint, obsolete)
+ }
+ if recovery == "fallback-late":
+ for name in ("scan-manifest.json", "findings.json", "coverage.json"):
+ (scan.scan_dir / name).unlink()
+ obsolete.unlink()
+ del original_sources[obsolete]
+ with monkeypatch.context() as interrupted:
+ if retry_publication:
+
+ def fail_publication(*args, **kwargs):
+ raise OSError("Synthetic publication interruption.")
+
+ interrupted.setattr(
+ workbench_api["saved_results"],
+ "_write_prepared_scan_finalization",
+ fail_publication,
+ )
+ stopped = workbench_api["saved_results"].fail_scan(
+ workbench_api["_WORKBENCH_DB_CONTEXT"],
+ workbench_db,
+ Namespace(scan_id=scan.scan_id, claim_token=None, cost_json=None, message="Stopped."),
+ )["scan"]
+ assert stopped["resultsRecoveryNeeded"] is retry_publication
+ frozen_sources = {
+ path: path.read_bytes() for path in (scan.scan_dir / "checkpoints").glob("*.json")
+ }
+ if recovery == "missing":
+ for name in ("scan-manifest.json", "findings.json", "coverage.json"):
+ (scan.scan_dir / name).unlink()
+ elif recovery == "changed":
+ (scan.scan_dir / "coverage.json").write_text(
+ json.dumps({**scan.coverage, "openQuestions": ["Late parent content must be ignored."]})
+ )
+ elif recovery in {"late", "fallback-late"}:
+ late = write_checkpoint(
+ scan.scan_dir / "checkpoints",
+ {
+ "scanId": scan.scan_id,
+ "complete": False,
+ "findings": [],
+ "coverage": {
+ **scan.coverage,
+ "completeness": "partial",
+ "openQuestions": ["Late proof gap."],
+ },
+ },
+ )
+ original_sources[late] = late.read_bytes()
+ recovered = workbench_api["saved_results"].recover_scan_results(
+ workbench_api["_WORKBENCH_DB_CONTEXT"], workbench_db, Namespace(scan_id=scan.scan_id)
+ )["scan"]
+ assert recovered["resultsRecoveryNeeded"] is False
+ coverage = json.loads((scan.scan_dir / "coverage.json").read_text())
+ expected_questions = [] if recovery == "fallback-late" else [question]
+ if recovery in {"late", "fallback-late"}:
+ expected_questions.append({"question": "Late proof gap."})
+ if recovery == "incomplete":
+ expected_questions.append(
+ {"question": "This question was answered by the final parent draft."}
+ )
+ assert coverage["openQuestions"] == expected_questions
+ assert coverage.get("reviews", []) == reviews
+ assert [item["id"] for item in coverage["surfaces"]] == (
+ ["old-disposition"] if recovery == "incomplete" else []
+ )
+ assert (
+ len([item for item in coverage["deferred"] if item.get("reason") == deferred["reason"]])
+ == 1
+ )
+ assert all(path.read_bytes() == original for path, original in original_sources.items())
+ assert all(path.read_bytes() == original for path, original in frozen_sources.items())
+ published = (scan.scan_dir / "coverage.json").read_bytes()
+ workbench_api["saved_results"].recover_scan_results(
+ workbench_api["_WORKBENCH_DB_CONTEXT"], workbench_db, Namespace(scan_id=scan.scan_id)
+ )
+ assert (scan.scan_dir / "coverage.json").read_bytes() == published
+
+
[email protected]("select_head", [False, True])
+def test_matching_accepted_checkpoint_preserves_receipt_path(
+ workbench_api, workbench_db, publication_scan, select_head
+):
+ scan = publication_scan()
+ result = add_worker(workbench_db, scan)
+ worker_id = result.parent.name
+ result = (
+ scan.scan_dir
+ / "artifacts"
+ / "deep_discovery"
+ / "workers"
+ / worker_id
+ / "output"
+ / "result.json"
+ )
+ result.parent.mkdir(parents=True)
+ with workbench_db:
+ workbench_db.execute(
+ "UPDATE deep_scan_workers SET artifact_dir = ?, result_manifest_path = ? WHERE id = ?",
+ (str(result.parent), str(result), worker_id),
+ )
+ receipt = result.parent / "artifacts" / "evidence.txt"
+ receipt.parent.mkdir(parents=True, exist_ok=True)
+ receipt.write_text("Synthetic source-review evidence.")
+ draft = {
+ "scanId": scan.scan_id,
+ "complete": True,
+ "findings": [],
+ "coverage": {
+ **scan.coverage,
+ "completeness": "complete",
+ "deferred": [],
+ "surfaces": [
+ {
+ "id": "source-review",
+ "label": "Accepted source review",
+ "disposition": "no_issue_found",
+ "receiptRefs": ["artifacts/evidence.txt"],
+ }
+ ],
+ },
+ }
+ result.write_text(json.dumps(draft))
+ checkpoint = write_checkpoint(result.parent / "checkpoints", draft)
+ os.utime(result, ns=(100, 100))
+ os.utime(checkpoint, ns=(150, 150))
+ head = result.parent / "checkpoint-head.json"
+ if select_head:
+ head.write_text(json.dumps({"checkpoint": checkpoint.name}))
+ os.utime(head, ns=(200, 200))
+ for name in ("scan-manifest.json", "findings.json", "coverage.json"):
+ (scan.scan_dir / name).unlink()
+ workbench_api["saved_results"].fail_scan(
+ workbench_api["_WORKBENCH_DB_CONTEXT"],
+ workbench_db,
+ Namespace(scan_id=scan.scan_id, claim_token=None, cost_json=None, message="Stopped."),
+ )
+ coverage = json.loads((scan.scan_dir / "coverage.json").read_text())
+ matching = [row for row in coverage["surfaces"] if row.get("label") == "Accepted source review"]
+ assert matching
+ assert all(
+ row["receiptRefs"] == [receipt.relative_to(scan.scan_dir).as_posix()] for row in matching
+ )
+ assert len(matching) == 1
+ assert matching[0]["disposition"] == "no_issue_found"
+ assert receipt.read_text() == "Synthetic source-review evidence."
+
+
[email protected]("retry_publication", [False, True])
+def test_accepted_worker_missing_completeness_keeps_valid_findings(
+ workbench_api, workbench_db, publication_scan, monkeypatch, retry_publication
+):
+ scan = publication_scan()
+ result = add_worker(workbench_db, scan)
+ worker_id = result.parent.name
+ finding = copy.deepcopy(scan.findings[0])
+ finding["summary"] = "Synthetic accepted worker evidence."
+ finding["identity"] = {"anchor": "accepted-worker-report"}
+ result.write_text(
+ json.dumps(
+ {
+ "scanId": scan.scan_id,
+ "complete": True,
+ "findings": [finding],
+ "coverage": {
+ key: value for key, value in scan.coverage.items() if key != "completeness"
+ },
+ }
+ )
+ )
+ original = result.read_bytes()
+ with monkeypatch.context() as interrupted:
+ if retry_publication:
+
+ def fail_publication(*args, **kwargs):
+ raise OSError("Synthetic publication interruption.")
+
+ interrupted.setattr(
+ workbench_api["saved_results"],
+ "_write_prepared_scan_finalization",
+ fail_publication,
+ )
+ stopped = workbench_api["saved_results"].fail_scan(
+ workbench_api["_WORKBENCH_DB_CONTEXT"],
+ workbench_db,
+ Namespace(scan_id=scan.scan_id, claim_token=None, cost_json=None, message="Stopped."),
+ )["scan"]
+ assert stopped["resultsRecoveryNeeded"] is retry_publication
+ recovered = workbench_api["saved_results"].recover_scan_results(
+ workbench_api["_WORKBENCH_DB_CONTEXT"], workbench_db, Namespace(scan_id=scan.scan_id)
+ )["scan"]
+ assert recovered["resultsRecoveryNeeded"] is False
+ findings = json.loads((scan.scan_dir / "findings.json").read_text())["findings"]
+ assert any(row["summary"] == finding["summary"] for row in findings)
+ coverage = json.loads((scan.scan_dir / "coverage.json").read_text())
+ assert {"workerId": worker_id, "attempt": 1, "completeness": "partial"} in coverage["reviews"]
+ assert "completeness" not in coverage
+ assert result.read_bytes() == original
+
+
[email protected]("retry_publication", [False, True])
+def test_idless_accepted_surface_matches_projection_after_checkpoint_id_inference(
+ workbench_api, workbench_db, publication_scan, monkeypatch, retry_publication
+):
+ scan = publication_scan()
+ result = add_worker(workbench_db, scan)
+ worker_id = result.parent.name
+ surface = {
+ "label": "Accepted source review",
+ "disposition": "no_issue_found",
+ "receiptRefs": [],
+ }
+ draft = {
+ "scanId": scan.scan_id,
+ "complete": True,
+ "findings": [],
+ "coverage": {**scan.coverage, "surfaces": [surface]},
+ }
+ result.write_text(json.dumps(draft))
+ checkpoint = write_checkpoint(
+ result.parent / "checkpoints",
+ {
+ **draft,
+ "complete": False,
+ "coverage": {**scan.coverage, "surfaces": [{**surface, "id": "older-source-id"}]},
+ },
+ )
+ os.utime(checkpoint, ns=(100, 100))
+ os.utime(result, ns=(200, 200))
+ projected = {
+ **surface,
+ "id": f"{worker_id}-attempt-1-surface-1",
+ "provenance": {"workerId": worker_id, "attempt": 1},
+ }
+ (scan.scan_dir / "coverage.json").write_text(
+ json.dumps(
+ {
+ **scan.coverage,
+ "surfaces": [projected],
+ "reviews": [{"workerId": worker_id, "attempt": 1, "completeness": "complete"}],
+ }
+ )
+ )
+ originals = {path: path.read_bytes() for path in (result, checkpoint)}
+ with monkeypatch.context() as interrupted:
+ if retry_publication:
+
+ def fail_publication(*args, **kwargs):
+ raise OSError("Synthetic publication interruption.")
+
+ interrupted.setattr(
+ workbench_api["saved_results"],
+ "_write_prepared_scan_finalization",
+ fail_publication,
+ )
+ stopped = workbench_api["saved_results"].fail_scan(
+ workbench_api["_WORKBENCH_DB_CONTEXT"],
+ workbench_db,
+ Namespace(scan_id=scan.scan_id, claim_token=None, cost_json=None, message="Stopped."),
+ )["scan"]
+ assert stopped["resultsRecoveryNeeded"] is retry_publication
+ recovered = workbench_api["saved_results"].recover_scan_results(
+ workbench_api["_WORKBENCH_DB_CONTEXT"], workbench_db, Namespace(scan_id=scan.scan_id)
+ )["scan"]
+ assert recovered["resultsRecoveryNeeded"] is False
+ coverage_path = scan.scan_dir / "coverage.json"
+ coverage = json.loads(coverage_path.read_text())
+ assert coverage["surfaces"] == [projected]
+ published = coverage_path.read_bytes()
+ workbench_api["saved_results"].recover_scan_results(
+ workbench_api["_WORKBENCH_DB_CONTEXT"], workbench_db, Namespace(scan_id=scan.scan_id)
+ )
+ assert coverage_path.read_bytes() == published
+ assert all(path.read_bytes() == original for path, original in originals.items())
+
+
[email protected]("claimed_worker", [False, True])
[email protected]("retry_publication", [False, True])
[email protected]("selected_checkpoint", [False, True])
+def test_parent_candidate_provenance_cannot_override_accepted_worker_evidence(
+ workbench_api,
+ workbench_db,
+ publication_scan,
+ monkeypatch,
+ claimed_worker,
+ retry_publication,
+ selected_checkpoint,
+):
+ scan = publication_scan()
+ result = add_worker(workbench_db, scan)
+ worker_id = result.parent.name
+ finding = copy.deepcopy(scan.findings[0])
+ finding["provenance"]["candidateId"] = "source-candidate"
+ finding["summary"] = "Accepted independent source evidence."
+ result.write_text(
+ json.dumps(
+ {
+ "scanId": scan.scan_id,
+ "complete": True,
+ "findings": [finding],
+ "coverage": scan.coverage,
+ }
+ )
+ )
+ originals = {result: result.read_bytes()}
+ if selected_checkpoint:
+ checkpoint = write_checkpoint(result.parent / "checkpoints", json.loads(result.read_text()))
+ head = result.parent / "checkpoint-head.json"
+ head.write_text(json.dumps({"checkpoint": checkpoint.name}))
+ pending_result = json.loads(result.read_text())
+ pending_result["findings"] = []
+ pending_result["coverage"] = {
+ **scan.coverage,
+ "completeness": "partial",
+ "deferred": [
+ {
+ "id": "source-review",
+ "candidateId": "source-candidate",
+ "reason": "Verify source evidence.",
+ }
+ ],
+ }
+ result.write_text(json.dumps(pending_result))
+ os.utime(result, ns=(100, 100))
+ os.utime(checkpoint, ns=(300, 300))
+ os.utime(head, ns=(300, 300))
+ originals = {path: path.read_bytes() for path in (result, checkpoint, head)}
+ rejection = {
+ "id": "parent-review",
+ "candidateId": "source-candidate",
+ "label": "Parent candidate assessment",
+ "disposition": "rejected",
+ "receiptRefs": [],
+ }
+ if claimed_worker:
+ rejection["provenance"] = {
+ "workerId": worker_id,
+ "attempt": 1,
+ "candidateId": "source-candidate",
+ }
+ (scan.scan_dir / "findings.json").write_text(json.dumps({"findings": []}))
+ (scan.scan_dir / "coverage.json").write_text(
+ json.dumps(
+ {
+ **scan.coverage,
+ "surfaces": [rejection],
+ }
+ )
+ )
+ os.utime(scan.scan_dir / "coverage.json", ns=(400, 400))
+ with monkeypatch.context() as interrupted:
+ if retry_publication:
+
+ def fail_publication(*args, **kwargs):
+ raise OSError("Synthetic publication interruption.")
+
+ interrupted.setattr(
+ workbench_api["saved_results"],
+ "_write_prepared_scan_finalization",
+ fail_publication,
+ )
+ stopped = workbench_api["saved_results"].fail_scan(
+ workbench_api["_WORKBENCH_DB_CONTEXT"],
+ workbench_db,
+ Namespace(scan_id=scan.scan_id, claim_token=None, cost_json=None, message="Stopped."),
+ )["scan"]
+ assert stopped["resultsRecoveryNeeded"] is retry_publication
+ recovered = workbench_api["saved_results"].recover_scan_results(
+ workbench_api["_WORKBENCH_DB_CONTEXT"], workbench_db, Namespace(scan_id=scan.scan_id)
+ )["scan"]
+ assert recovered["resultsRecoveryNeeded"] is False
+ findings = json.loads((scan.scan_dir / "findings.json").read_text())["findings"]
+ assert [row["summary"] for row in findings] == [finding["summary"]]
+ assert all(path.read_bytes() == contents for path, contents in originals.items())
+
+
[email protected]("other_worker", [False, True])
[email protected]("retry_publication", [False, True])
+def test_projected_generic_closure_updates_parent_copy_in_its_worker_namespace(
+ workbench_api, workbench_db, publication_scan, monkeypatch, other_worker, retry_publication
+):
+ scan = publication_scan()
+ result = add_worker(workbench_db, scan)
+ worker_id = result.parent.name
+ surface = {
+ "id": "source-surface",
+ "label": "Source follow-up",
+ "disposition": "needs_follow_up",
+ "receiptRefs": [],
+ }
+ task = {
+ "id": "source-task",
+ "reason": "Complete source follow-up.",
+ "surfaceIds": [surface["id"]],
+ }
+ draft = {
+ "scanId": scan.scan_id,
+ "complete": True,
+ "findings": [],
+ "coverage": {
+ **scan.coverage,
+ "completeness": "partial",
+ "surfaces": [surface],
+ "deferred": [task],
+ },
+ }
+ result.write_text(json.dumps(draft))
+ projected_surface = {
+ **surface,
+ "id": f"{worker_id}-attempt-1-surface-1",
+ "provenance": {"workerId": worker_id, "attempt": 1, "sourceId": surface["id"]},
+ }
+ projected_task = {
+ **task,
+ "id": f"{worker_id}-attempt-1-deferred-1",
+ "surfaceIds": [projected_surface["id"]],
+ "provenance": {"workerId": worker_id, "attempt": 1, "sourceId": task["id"]},
+ }
+ (scan.scan_dir / "findings.json").write_text(json.dumps({"findings": []}))
+ parent_coverage = scan.scan_dir / "coverage.json"
+ parent_coverage.write_text(
+ json.dumps(
+ {
+ **scan.coverage,
+ "completeness": "partial",
+ "surfaces": [projected_surface],
+ "deferred": [projected_task],
+ "reviews": [{"workerId": worker_id, "attempt": 1, "completeness": "partial"}],
+ }
+ )
+ )
+ closure_result = add_worker(workbench_db, scan) if other_worker else result
+ if other_worker:
+ closure_result.write_text(json.dumps({**draft, "coverage": scan.coverage}))
+ closure = {
+ **draft,
+ "coverage": {
+ **scan.coverage,
+ "surfaces": [{**surface, "disposition": "no_issue_found"}],
+ "deferred": [],
+ "resolvedDeferred": [{"id": task["id"], "reason": "Source follow-up completed."}],
+ },
+ }
+ checkpoint = write_checkpoint(closure_result.parent / "checkpoints", closure)
+ os.utime(result, ns=(100, 100))
+ os.utime(parent_coverage, ns=(200, 200))
+ os.utime(checkpoint, ns=(300, 300))
+ (closure_result.parent / "checkpoint-head.json").write_text(
+ json.dumps({"checkpoint": checkpoint.name})
+ )
+ originals = {path: path.read_bytes() for path in {result, closure_result, checkpoint}}
+ with monkeypatch.context() as interrupted:
+ if retry_publication:
+
+ def fail_publication(*args, **kwargs):
+ raise OSError("Synthetic publication interruption.")
+
+ interrupted.setattr(
+ workbench_api["saved_results"],
+ "_write_prepared_scan_finalization",
+ fail_publication,
+ )
+ stopped = workbench_api["saved_results"].fail_scan(
+ workbench_api["_WORKBENCH_DB_CONTEXT"],
+ workbench_db,
+ Namespace(scan_id=scan.scan_id, claim_token=None, cost_json=None, message="Stopped."),
+ )["scan"]
+ assert stopped["resultsRecoveryNeeded"] is retry_publication
+ workbench_api["saved_results"].recover_scan_results(
+ workbench_api["_WORKBENCH_DB_CONTEXT"], workbench_db, Namespace(scan_id=scan.scan_id)
+ )
+ coverage = json.loads(parent_coverage.read_text())
+ pending = [row for row in coverage["deferred"] if row.get("reason") == task["reason"]]
+ assert len(pending) == int(other_worker)
+ source_surfaces = [
+ row
+ for row in coverage["surfaces"]
+ if row.get("provenance", {}).get("workerId") == worker_id
+ ]
+ assert len(source_surfaces) == 1
+ assert source_surfaces[0]["disposition"] == (
+ "needs_follow_up" if other_worker else "no_issue_found"
+ )
+ assert all(path.read_bytes() == value for path, value in originals.items())
+ published = parent_coverage.read_bytes()
+ workbench_api["saved_results"].recover_scan_results(
+ workbench_api["_WORKBENCH_DB_CONTEXT"], workbench_db, Namespace(scan_id=scan.scan_id)
+ )
+ assert parent_coverage.read_bytes() == published
+
+
[email protected]("disposition", ["rejected", "reported"])
[email protected]("retry_publication", [False, True])
[email protected]("parent_review", ["earlier", "copied-later", "changed-later"])
+def test_selected_candidate_outcome_removes_its_projected_parent_pending_rows(
+ workbench_api,
+ workbench_db,
+ publication_scan,
+ monkeypatch,
+ disposition,
+ retry_publication,
+ parent_review,
+):
+ scan = publication_scan()
+ result = add_worker(workbench_db, scan)
+ worker_id = result.parent.name
+ candidate = "source-candidate"
+ surface = {
+ "id": "source-surface",
+ "candidateId": candidate,
+ "label": "Candidate follow-up",
+ "disposition": "needs_follow_up",
+ "receiptRefs": [],
+ }
+ task = {
+ "id": "source-task",
+ "candidateId": candidate,
+ "reason": "Complete candidate follow-up.",
+ }
+ draft = {
+ "scanId": scan.scan_id,
+ "complete": True,
+ "findings": [],
+ "coverage": {
+ **scan.coverage,
+ "completeness": "partial",
+ "surfaces": [surface],
+ "deferred": [task],
+ },
+ }
+ result.write_text(json.dumps(draft))
+ projected_candidate = (
+ f"{worker_id}-attempt-1-candidate-{hashlib.sha256(candidate.encode()).hexdigest()}"
+ )
+ projected = {
+ field: [
+ {
+ **row,
+ "id": f"{worker_id}-attempt-1-{name}-1",
+ "candidateId": projected_candidate,
+ "provenance": {
+ "workerId": worker_id,
+ "attempt": 1,
+ "sourceId": row["id"],
+ "candidateId": candidate,
+ },
+ }
+ ]
+ for field, name, row in (("surfaces", "surface", surface), ("deferred", "deferred", task))
+ }
+ if parent_review == "changed-later":
+ projected["surfaces"][0]["label"] = "Parent requested additional review."
+ projected["deferred"][0]["reason"] = "Parent requested additional proof."
+ (scan.scan_dir / "findings.json").write_text(json.dumps({"findings": []}))
+ coverage_path = scan.scan_dir / "coverage.json"
+ coverage_path.write_text(
+ json.dumps(
+ {
+ **scan.coverage,
+ "completeness": "partial",
+ **projected,
+ "reviews": [{"workerId": worker_id, "attempt": 1, "completeness": "partial"}],
+ }
+ )
+ )
+ finding = copy.deepcopy(scan.findings[0])
+ finding["provenance"]["candidateId"] = candidate
+ checkpoint = write_checkpoint(
+ result.parent / "checkpoints",
+ {
+ **draft,
+ "findings": [finding] if disposition == "reported" else [],
+ "coverage": {
+ **scan.coverage,
+ "surfaces": [{**surface, "disposition": disposition}],
+ "deferred": [],
+ },
+ },
+ )
+ os.utime(result, ns=(100, 100))
+ modified = 200 if parent_review == "earlier" else 400
+ os.utime(coverage_path, ns=(modified, modified))
+ os.utime(checkpoint, ns=(300, 300))
+ head = result.parent / "checkpoint-head.json"
+ head.write_text(json.dumps({"checkpoint": checkpoint.name}))
+ os.utime(head, ns=(300, 300))
+ originals = {path: path.read_bytes() for path in (result, checkpoint)}
+ with monkeypatch.context() as interrupted:
+ if retry_publication:
+
+ def fail_publication(*args, **kwargs):
+ raise OSError("Synthetic publication interruption.")
+
+ interrupted.setattr(
+ workbench_api["saved_results"],
+ "_write_prepared_scan_finalization",
+ fail_publication,
+ )
+ stopped = workbench_api["saved_results"].fail_scan(
+ workbench_api["_WORKBENCH_DB_CONTEXT"],
+ workbench_db,
+ Namespace(scan_id=scan.scan_id, claim_token=None, cost_json=None, message="Stopped."),
+ )["scan"]
+ assert stopped["resultsRecoveryNeeded"] is retry_publication
+ workbench_api["saved_results"].recover_scan_results(
+ workbench_api["_WORKBENCH_DB_CONTEXT"], workbench_db, Namespace(scan_id=scan.scan_id)
+ )
+ coverage = json.loads(coverage_path.read_text())
+ if parent_review != "changed-later":
+ assert not any(row.get("reason") == task["reason"] for row in coverage["deferred"])
+ assert any(
+ row.get("reason") == "Parent requested additional proof." for row in coverage["deferred"]
+ ) is (parent_review == "changed-later")
+ assert any(
+ row.get("disposition") == "needs_follow_up"
+ and row.get("provenance", {}).get("workerId") == worker_id
+ for row in coverage["surfaces"]
+ ) is (parent_review == "changed-later")
+ findings = json.loads((scan.scan_dir / "findings.json").read_text())["findings"]
+ assert len(findings) == int(disposition == "reported")
+ assert all(path.read_bytes() == value for path, value in originals.items())
+ published = coverage_path.read_bytes()
+ workbench_api["saved_results"].recover_scan_results(
+ workbench_api["_WORKBENCH_DB_CONTEXT"], workbench_db, Namespace(scan_id=scan.scan_id)
+ )
+ assert coverage_path.read_bytes() == published
+
+
[email protected]("retry_publication", [False, True])
+ "change",
+ [
+ "new linked surface",
+ "revised linked surface",
+ "reopened accepted surface",
+ "reopened mixed receipts",
+ "candidate id fallback",
+ "closed sibling candidate",
+ "mixed receipts",
+ "repeated pending",
+ "unchanged",
+ ],
+)
+def test_selected_worker_projection_keeps_links_and_pending_authority(
+ workbench_api, workbench_db, publication_scan, monkeypatch, change, retry_publication
+):
+ scan = publication_scan()
+ result = add_worker(workbench_db, scan)
+ worker_id = result.parent.name
+ prefix = f"{worker_id}-attempt-1"
+ candidate = "source-candidate"
+ surface = {
+ "id": "source-surface",
+ "label": "Source follow-up",
+ "disposition": "needs_follow_up",
+ "receiptRefs": [],
+ }
+ task = {
+ "id": "source-task",
+ "reason": "Complete source follow-up.",
+ "surfaceIds": [surface["id"]],
+ }
+ surfaces, deferred = [surface], [task]
+ if change == "candidate id fallback":
+ task.update(id=candidate, candidate=copy.deepcopy(scan.findings[0]))
+ surface["candidateId"] = candidate
+ elif change == "closed sibling candidate":
+ deferred.append({**task, "id": "candidate-task", "candidateId": candidate})
+ elif change in {"new linked surface", "reopened accepted surface", "reopened mixed receipts"}:
+ surface["disposition"] = "no_issue_found"
+ deferred = []
+ if change in {"mixed receipts", "reopened mixed receipts"}:
+ result = (
+ scan.scan_dir
+ / "artifacts"
+ / "deep_discovery"
+ / "workers"
+ / worker_id
+ / "output"
+ / "result.json"
+ )
+ result.parent.mkdir(parents=True)
+ with workbench_db:
+ workbench_db.execute(
+ "UPDATE deep_scan_workers SET artifact_dir = ?, result_manifest_path = ? WHERE id = ?",
+ (str(result.parent), str(result), worker_id),
+ )
+ surface["receiptRefs"] = ["artifacts/old.txt"]
+ for name in ("old", "new"):
+ receipt = result.parent / "artifacts" / f"{name}.txt"
+ receipt.parent.mkdir(exist_ok=True)
+ receipt.write_text(f"Synthetic {name} receipt.\n")
+ draft = {
+ "scanId": scan.scan_id,
+ "complete": True,
+ "findings": [],
+ "coverage": {
+ **scan.coverage,
+ "completeness": "partial",
+ "surfaces": surfaces,
+ "deferred": deferred,
+ },
+ }
+ if change == "reopened mixed receipts":
+ draft["coverage"]["resolvedDeferred"] = [
+ {"id": task["id"], "reason": "Earlier source follow-up completed."}
+ ]
+ result.write_text(json.dumps(draft))
+ projected = {}
+ for field, name, rows in (
+ ("surfaces", "surface", surfaces),
+ ("deferred", "deferred", deferred),
+ ):
+ projected[field] = []
+ for index, original in enumerate(rows, 1):
+ row = copy.deepcopy(original)
+ row["id"] = f"{prefix}-{name}-{index}"
+ row["provenance"] = {"workerId": worker_id, "attempt": 1, "sourceId": original["id"]}
+ if "candidateId" in original:
+ row["provenance"]["candidateId"] = original["candidateId"]
+ row["candidateId"] = (
+ f"{prefix}-candidate-{hashlib.sha256(candidate.encode()).hexdigest()}"
+ )
+ if field == "surfaces":
+ row["receiptRefs"] = [
+ (result.parent / ref).relative_to(scan.scan_dir).as_posix()
+ for ref in original["receiptRefs"]
+ ]
+ else:
+ row["surfaceIds"] = [f"{prefix}-surface-1"]
+ projected[field].append(row)
+ coverage_path = scan.scan_dir / "coverage.json"
+ (scan.scan_dir / "findings.json").write_text(json.dumps({"findings": []}))
+ coverage_path.write_text(
+ json.dumps(
+ {
+ **scan.coverage,
+ "completeness": "partial",
+ **projected,
+ "reviews": [{"workerId": worker_id, "attempt": 1, "completeness": "partial"}],
+ }
+ )
+ )
+ checkpoint = None
+ if change != "unchanged":
+ updated = copy.deepcopy(draft)
+ if change in {
+ "new linked surface",
+ "revised linked surface",
+ "reopened accepted surface",
+ "reopened mixed receipts",
+ }:
+ updated_id = "new-surface" if change == "new linked surface" else surface["id"]
+ updated_task = (
+ task["id"]
+ if change in {"revised linked surface", "reopened mixed receipts"}
+ else "new-task"
+ )
+ updated["coverage"].update(
+ surfaces=[
+ {
+ **surface,
+ "id": updated_id,
+ "label": "New follow-up",
+ "disposition": "needs_follow_up",
+ }
+ ],
+ deferred=[{**task, "id": updated_task, "surfaceIds": [updated_id]}],
+ )
+ if change == "reopened mixed receipts":
+ updated["coverage"].pop("resolvedDeferred")
+ updated["coverage"]["surfaces"][0]["receiptRefs"] = ["artifacts/new.txt"]
+ elif change == "candidate id fallback":
+ updated["coverage"].update(
+ surfaces=[{**surface, "disposition": "rejected"}], deferred=[]
+ )
+ elif change in {"closed sibling candidate", "mixed receipts"}:
+ updated["coverage"].update(
+ surfaces=[
+ {
+ **surface,
+ "disposition": "no_issue_found",
+ "receiptRefs": ["artifacts/new.txt"] if change == "mixed receipts" else [],
+ }
+ ],
+ deferred=[],
+ resolvedDeferred=[{"id": task["id"], "reason": "Source follow-up completed."}],
+ )
+ if change == "closed sibling candidate":
+ updated["coverage"]["surfaces"].append(
+ {
+ **surface,
+ "id": "candidate-surface",
+ "candidateId": candidate,
+ "label": "Candidate proof",
+ "disposition": "rejected",
+ }
+ )
+ checkpoint = write_checkpoint(result.parent / "checkpoints", updated)
+ os.utime(checkpoint, ns=(300, 300))
+ (result.parent / "checkpoint-head.json").write_text(
+ json.dumps({"checkpoint": checkpoint.name})
+ )
+ os.utime(result, ns=(100, 100))
+ os.utime(coverage_path, ns=(200, 200))
+ if change == "reopened mixed receipts":
+ result.write_text(json.dumps(updated))
+ checkpoint.write_text(json.dumps(draft))
+ (result.parent / "checkpoint-head.json").unlink()
+ os.utime(checkpoint, ns=(100, 100))
+ os.utime(result, ns=(300, 300))
+ originals = {
+ path: path.read_bytes() for path in [result] + ([checkpoint] if checkpoint else [])
+ }
+ with monkeypatch.context() as interrupted:
+ if retry_publication:
+
+ def fail_publication(*args, **kwargs):
+ raise OSError("Synthetic publication interruption.")
+
+ interrupted.setattr(
+ workbench_api["saved_results"],
+ "_write_prepared_scan_finalization",
+ fail_publication,
+ )
+ stopped = workbench_api["saved_results"].fail_scan(
+ workbench_api["_WORKBENCH_DB_CONTEXT"],
+ workbench_db,
+ Namespace(scan_id=scan.scan_id, claim_token=None, cost_json=None, message="Stopped."),
+ )["scan"]
+ assert stopped["resultsRecoveryNeeded"] is retry_publication
+ workbench_api["saved_results"].recover_scan_results(
+ workbench_api["_WORKBENCH_DB_CONTEXT"], workbench_db, Namespace(scan_id=scan.scan_id)
+ )
+ coverage = json.loads(coverage_path.read_text())
+ pending = [row for row in coverage["deferred"] if row.get("reason") == task["reason"]]
+ assert len(pending) == int(
+ change
+ in {
+ "new linked surface",
+ "revised linked surface",
+ "reopened accepted surface",
+ "reopened mixed receipts",
+ "repeated pending",
+ "unchanged",
+ }
+ )
+ if change in {
+ "new linked surface",
+ "revised linked surface",
+ "reopened accepted surface",
+ "reopened mixed receipts",
+ }:
+ linked = next(
+ row for row in coverage["surfaces"] if row["id"] == pending[0]["surfaceIds"][0]
+ )
+ assert linked["label"] == "New follow-up"
+ if change == "reopened mixed receipts":
+ assert set(linked["receiptRefs"]) == {
+ (result.parent / "artifacts" / f"{name}.txt").relative_to(scan.scan_dir).as_posix()
+ for name in ("old", "new")
+ }
+ assert len({row["id"] for row in coverage["surfaces"]}) == len(coverage["surfaces"])
+ elif change in {"closed sibling candidate", "mixed receipts"}:
+ saved_surface = next(
+ row for row in coverage["surfaces"] if row.get("label") == surface["label"]
+ )
+ assert saved_surface["disposition"] == "no_issue_found", json.dumps(saved_surface)
+ if change == "mixed receipts":
+ assert set(saved_surface["receiptRefs"]) == {
+ (result.parent / "artifacts" / f"{name}.txt").relative_to(scan.scan_dir).as_posix()
+ for name in ("old", "new")
+ }
+ assert all(path.read_bytes() == content for path, content in originals.items())
+ published = coverage_path.read_bytes()
+ workbench_api["saved_results"].recover_scan_results(
+ workbench_api["_WORKBENCH_DB_CONTEXT"], workbench_db, Namespace(scan_id=scan.scan_id)
+ )
+ assert coverage_path.read_bytes() == published
+
+
+def publish_review_projection(workbench_api, connection, scan, coverage):
+ # Publish through an active coordinator before exercising stopped recovery.
+ with connection:
+ connection.execute(
+ "UPDATE deep_scan_runs SET status = 'running', coordinator_generation = 2, "
+ "terminal_reason = NULL, completed_at = NULL WHERE scan_id = ?",
+ (scan.scan_id,),
+ )
+ documents = {
+ "deepScanPublication": {"coordinatorGeneration": 2, "resultPath": None},
+ "manifest": json.loads((scan.scan_dir / "scan-manifest.json").read_text()),
+ "findings": {"findings": []},
+ "coverage": coverage,
+ }
+ staged = scan.scan_dir / "drafts" / f"{uuid.uuid4()}.json"
+ staged.parent.mkdir(exist_ok=True)
+ staged.write_text(json.dumps(documents))
+ saved = workbench_api["saved_results"]
+ saved.write_scan_draft(
+ workbench_api["_WORKBENCH_DB_CONTEXT"],
+ connection,
+ Namespace(
+ scan_id=scan.scan_id,
+ claim_token=None,
+ draft_path=str(staged),
+ checkpoint_path=None,
+ expected_draft_digest=None,
+ ),
+ )
+
+
+def stop_and_recover_projection(workbench_api, connection, scan, monkeypatch, retry):
+ saved = workbench_api["saved_results"]
+ context = workbench_api["_WORKBENCH_DB_CONTEXT"]
+ with monkeypatch.context() as interrupted:
+ if retry:
+
+ def fail_publication(*args, **kwargs):
+ raise OSError("Synthetic publication interruption.")
+
+ interrupted.setattr(saved, "_write_prepared_scan_finalization", fail_publication)
+ saved.fail_scan(
+ context,
+ connection,
+ Namespace(scan_id=scan.scan_id, claim_token=None, cost_json=None, message="Stopped."),
+ )
+ saved.recover_scan_results(context, connection, Namespace(scan_id=scan.scan_id))
+ published = (scan.scan_dir / "coverage.json").read_bytes()
+ saved.recover_scan_results(context, connection, Namespace(scan_id=scan.scan_id))
+ assert (scan.scan_dir / "coverage.json").read_bytes() == published
+ return json.loads(published)
+
+
+ "receipt_ref",
+ ["artifacts/review.txt", "artifacts/./review.txt", "scan-relative", "scan-relative-equivalent"],
+ ids=["canonical", "equivalent", "scan-relative", "scan-relative-equivalent"],
+)
[email protected]("closure", [False, True], ids=["unchanged", "generic-closure"])
[email protected]("retry", [False, True], ids=["direct", "failed-retry"])
+def test_generic_surface_receipt_union_keeps_each_source_directory(
+ workbench_api, workbench_db, publication_scan, monkeypatch, closure, retry, receipt_ref
+):
+ scan = publication_scan()
+ initial = add_worker(workbench_db, scan)
+ worker_id = initial.parent.name
+ output = scan.scan_dir / "artifacts" / worker_id / "output"
+ output.mkdir(parents=True)
+ result = output / "result.json"
+ with workbench_db:
+ workbench_db.execute(
+ "UPDATE deep_scan_workers SET artifact_dir = ?, result_manifest_path = ? WHERE id = ?",
+ (str(output), str(result), worker_id),
+ )
+ receipt = output / "artifacts" / "review.txt"
+ receipt.parent.mkdir()
+ receipt.write_text("Synthetic completed source review.\n")
+ if receipt_ref.startswith("scan-relative"):
+ canonical_ref = receipt.relative_to(scan.scan_dir).as_posix()
+ receipt_ref = (
+ canonical_ref.replace("artifacts/", "artifacts/./", 1)
+ if receipt_ref == "scan-relative-equivalent"
+ else canonical_ref
+ )
+ surface = {
+ "id": "review",
+ "label": "Completed source review",
+ "disposition": "no_issue_found",
+ "receiptRefs": [receipt_ref],
+ }
+ source = {**scan.coverage, "surfaces": [surface], "deferred": []}
+ if closure:
+ source["resolvedDeferred"] = [{"id": "review", "reason": "Source review completed."}]
+ result.write_text(
+ json.dumps({"scanId": scan.scan_id, "complete": True, "findings": [], "coverage": source})
+ )
+ original = result.read_bytes()
+ projected = {
+ **surface,
+ "id": f"{worker_id}-attempt-1-surface-1",
+ "receiptRefs": [receipt.relative_to(scan.scan_dir).as_posix()],
+ "provenance": {"workerId": worker_id, "attempt": 1, "sourceId": "review"},
+ }
+ publish_review_projection(
+ workbench_api,
+ workbench_db,
+ scan,
+ {
+ **scan.coverage,
+ "surfaces": [projected],
+ "deferred": [],
+ "reviews": [{"workerId": worker_id, "attempt": 1, "completeness": "complete"}],
+ },
+ )
+ coverage = stop_and_recover_projection(workbench_api, workbench_db, scan, monkeypatch, retry)
+ assert coverage["surfaces"] == [projected]
+ assert result.read_bytes() == original
+
+
[email protected]("spelling", ["unchanged", "equivalent", "scan relative", "omitted"])
[email protected]("parent", [False, True])
[email protected]("retry", [False, True])
+def test_linked_archived_receipts_preserve_accepted_surface_links(
+ workbench_api, workbench_db, publication_scan, monkeypatch, spelling, parent, retry
+):
+ scan = publication_scan()
+ initial = add_worker(workbench_db, scan)
+ worker_id = initial.parent.name
+ output = scan.scan_dir / "artifacts" / worker_id / "output"
+ output.mkdir(parents=True)
+ result = output / "result.json"
+ with workbench_db:
+ workbench_db.execute(
+ "UPDATE deep_scan_workers SET artifact_dir = ?, result_manifest_path = ? WHERE id = ?",
+ (str(output), str(result), worker_id),
+ )
+ receipt = output / "artifacts" / "review.txt"
+ receipt.parent.mkdir()
+ receipt.write_text("Synthetic completed review.\n")
+ surface = {
+ "id": "source-surface",
+ "label": "Source review",
+ "disposition": "needs_follow_up",
+ "receiptRefs": ["artifacts/review.txt"],
+ }
+ draft = {
+ "scanId": scan.scan_id,
+ "complete": True,
+ "findings": [],
+ "coverage": {
+ **scan.coverage,
+ "completeness": "partial",
+ "surfaces": [surface],
+ "deferred": [],
+ },
+ }
+ result.write_text(json.dumps(draft))
+ projected = {
+ **surface,
+ "id": f"{worker_id}-attempt-1-surface-1",
+ "receiptRefs": [receipt.relative_to(scan.scan_dir).as_posix()],
+ "provenance": {"workerId": worker_id, "attempt": 1, "sourceId": surface["id"]},
+ }
+ coverage_path = scan.scan_dir / "coverage.json"
+ (scan.scan_dir / "findings.json").write_text(json.dumps({"findings": []}))
+ coverage_path.write_text(
+ json.dumps(
+ {
+ **scan.coverage,
+ "surfaces": [projected],
+ "deferred": [],
+ "reviews": [{"workerId": worker_id, "attempt": 1, "completeness": "partial"}],
+ }
+ )
+ )
+ selected = copy.deepcopy(draft)
+ selected["complete"] = False
+ selected["coverage"]["deferred"] = [
+ {"id": "pending", "reason": "Verify accepted source review.", "surfaceIds": [surface["id"]]}
+ ]
+ if spelling == "equivalent":
+ selected["coverage"]["surfaces"][0]["receiptRefs"] = ["artifacts/./review.txt"]
+ elif spelling == "scan relative":
+ selected["coverage"]["surfaces"][0]["receiptRefs"] = [
+ receipt.relative_to(scan.scan_dir).as_posix()
+ ]
+ elif spelling == "omitted":
+ selected["coverage"]["surfaces"] = []
+ checkpoint = write_checkpoint(output / "checkpoints", selected)
+ head = output / "checkpoint-head.json"
+ head.write_text(json.dumps({"checkpoint": checkpoint.name}))
+ for path, stamp in ((result, 100), (coverage_path, 200), (checkpoint, 300), (head, 300)):
+ os.utime(path, ns=(stamp, stamp))
+ if not parent:
+ for name in ("scan-manifest.json", "findings.json", "coverage.json"):
+ (scan.scan_dir / name).unlink()
+ originals = {path: path.read_bytes() for path in (result, checkpoint, head)}
+ coverage = stop_and_recover_projection(workbench_api, workbench_db, scan, monkeypatch, retry)
+ pending = [
+ row for row in coverage["deferred"] if row.get("reason") == "Verify accepted source review."
+ ]
+ assert len(pending) == 1
+ assert pending[0]["surfaceIds"] == [projected["id"]]
+ assert coverage["surfaces"] == [projected]
+ assert all(path.read_bytes() == value for path, value in originals.items())
+
+
[email protected]("source", ["worker", "shared", "worker-collision"])
[email protected]("equivalent", [False, True])
[email protected]("parent", [False, True])
[email protected]("retry", [False, True])
+def test_shared_scan_receipts_preserve_existing_context_evidence(
+ workbench_api, workbench_db, publication_scan, monkeypatch, source, equivalent, parent, retry
+):
+ scan = publication_scan()
+ initial = add_worker(workbench_db, scan)
+ worker_id = initial.parent.name
+ output = scan.scan_dir / "artifacts" / worker_id / "output"
+ output.mkdir(parents=True)
+ result = output / "result.json"
+ with workbench_db:
+ workbench_db.execute(
+ "UPDATE deep_scan_workers SET artifact_dir = ?, result_manifest_path = ? WHERE id = ?",
+ (str(output), str(result), worker_id),
+ )
+ shared = source == "shared"
+ ref = "artifacts/01_context/false_positive_feedback.json" if shared else "artifacts/review.txt"
+ if source == "worker-collision":
+ parent_receipt = scan.scan_dir / ref
+ parent_receipt.parent.mkdir(exist_ok=True, parents=True)
+ parent_receipt.write_text("Synthetic unrelated parent review.\n")
+ receipt = (scan.scan_dir if shared else output) / ref
+ receipt.parent.mkdir(exist_ok=True, parents=True)
+ receipt.write_text("Synthetic completed source review.\n")
+ source_ref = ref.replace("artifacts/", "artifacts/./") if equivalent else ref
+ surface = {
+ "id": "review",
+ "label": "Completed source review",
+ "disposition": "no_issue_found",
+ "receiptRefs": [source_ref],
+ }
+ result.write_text(
+ json.dumps(
+ {
+ "scanId": scan.scan_id,
+ "complete": True,
+ "findings": [],
+ "coverage": {**scan.coverage, "surfaces": [surface], "deferred": []},
+ }
+ )
+ )
+ original = result.read_bytes()
+ projected = {
+ **surface,
+ "id": f"{worker_id}-attempt-1-surface-1",
+ "receiptRefs": [receipt.relative_to(scan.scan_dir).as_posix()],
+ "provenance": {"workerId": worker_id, "attempt": 1, "sourceId": "review"},
+ }
+ if parent:
+ publish_review_projection(
+ workbench_api,
+ workbench_db,
+ scan,
+ {
+ **scan.coverage,
+ "surfaces": [projected],
+ "deferred": [],
+ "reviews": [{"workerId": worker_id, "attempt": 1, "completeness": "complete"}],
+ },
+ )
+ else:
+ for name in ("scan-manifest.json", "findings.json", "coverage.json"):
+ (scan.scan_dir / name).unlink()
+ coverage = stop_and_recover_projection(workbench_api, workbench_db, scan, monkeypatch, retry)
+ assert coverage["surfaces"] == [projected]
+ assert result.read_bytes() == original
+
+
+def test_deferred_projection_preserves_each_referenced_source_surface(
+ workbench_api, workbench_db, publication_scan
+):
+ scan = publication_scan()
+ result = add_worker(workbench_db, scan)
+ worker_id = result.parent.name
+ surfaces = [
+ {
+ "id": f"source-{index}",
+ "label": f"Source boundary {index}",
+ "disposition": "needs_follow_up",
+ "receiptRefs": [],
+ }
+ for index in range(12)
+ ]
+ deferred = [
+ {"id": f"task-{index}", "reason": f"Verify source {index}.", "surfaceIds": [surface["id"]]}
+ for index, surface in enumerate(surfaces)
+ ]
+ draft = {
+ "scanId": scan.scan_id,
+ "complete": True,
+ "findings": [],
+ "coverage": {
+ **scan.coverage,
+ "completeness": "partial",
+ "surfaces": surfaces,
+ "deferred": deferred,
+ },
+ }
+ result.write_text(json.dumps(draft))
+ original = result.read_bytes()
+ for name in ("scan-manifest.json", "findings.json", "coverage.json"):
+ (scan.scan_dir / name).unlink()
+ workbench_api["saved_results"].fail_scan(
+ workbench_api["_WORKBENCH_DB_CONTEXT"],
+ workbench_db,
+ Namespace(scan_id=scan.scan_id, claim_token=None, cost_json=None, message="Stopped."),
+ )
+ coverage = json.loads((scan.scan_dir / "coverage.json").read_text())
+ projected = {row["provenance"]["sourceId"]: row for row in coverage["surfaces"]}
+ assert set(projected) == {surface["id"] for surface in surfaces}
+ pending = {
+ row["provenance"]["sourceId"]: row for row in coverage["deferred"] if row.get("surfaceIds")
+ }
+ assert set(pending) == {row["id"] for row in deferred}
+ for task in deferred:
+ row = pending[task["id"]]
+ assert row["surfaceIds"] == [projected[task["surfaceIds"][0]]["id"]]
+ assert row["provenance"]["workerId"] == worker_id
+ assert result.read_bytes() == original
diff --git a/plugins/codex-security/tests/test_recovered_worker_closeouts.py b/plugins/codex-security/tests/test_recovered_worker_closeouts.py
new file mode 100644
index 000000000..95c3711d3
--- /dev/null
+++ b/plugins/codex-security/tests/test_recovered_worker_closeouts.py
@@ -0,0 +1,131 @@
+from __future__ import annotations
+
+import json
+import os
+from pathlib import Path
+
+import pytest
+from workbench_test_support import (
+ replay_saved_results,
+ saved_binding,
+ saved_discovery_worker,
+ saved_draft,
+ write_checkpoint,
+)
+
+SCAN_ID = "worker-closeout-scan"
+CLOSURES = [
+ {"id": "review-a", "reason": "Source review completed."},
+ {"id": "review-b", "reason": "Source review completed."},
+]
+
+
+def observed(path: Path, seconds: int) -> None:
+ os.utime(path, ns=(seconds * 1_000_000_000, seconds * 1_000_000_000))
+
+
+def saved_worker(root: Path, worker_id: str, *, headed: bool = False):
+ output = root / worker_id / "output"
+ pending = saved_draft(
+ SCAN_ID, deferred=[{"id": row["id"], "reason": "Review remains."} for row in CLOSURES]
+ )
+ observed(write_checkpoint(output / "checkpoints", pending), 10)
+ complete = saved_draft(SCAN_ID, closures=CLOSURES, complete=True)
+ checkpoint = write_checkpoint(output / "checkpoints", complete)
+ observed(checkpoint, 20)
+ result = output / "result.json"
+ result.write_text(json.dumps(complete))
+ observed(result, 20)
+ if headed:
+ head = output / "checkpoint-head.json"
+ head.write_text(json.dumps({"checkpoint": checkpoint.name}))
+ observed(head, 20)
+ worker = saved_discovery_worker(output, worker_id, 2)
+ worker.update(status="succeeded", result_manifest_path=str(result))
+ return worker
+
+
+def parent_copy(root: Path, closures: list[dict]) -> None:
+ observed(
+ write_checkpoint(
+ root / "checkpoints", saved_draft(SCAN_ID, closures=closures, complete=True)
+ ),
+ 40,
+ )
+
+
+def recover(root: Path, module, workers):
+ binding = saved_binding("deep_repository", repository="synthetic")
+ documents = module.merge_saved_results(
+ root, SCAN_ID, binding, workers, [], stopped=True, reason="interrupted"
+ )
+ assert documents is not None
+ replay = replay_saved_results(module, documents, root, SCAN_ID, binding, workers)
+ assert replay is not None
+ assert replay[2] == documents[2]
+ return documents[2]
+
+
[email protected]("headed", [False, True])
[email protected]("published", [False, True])
+def test_worker_closeouts_keep_distinct_ids_and_do_not_duplicate_parent_copies(
+ tmp_path, workbench_api, headed, published
+):
+ workers = [
+ saved_worker(tmp_path, worker_id, headed=headed) for worker_id in ("first", "second")
+ ]
+ expected = [
+ {"id": f"{worker_id}-attempt-2-resolved-{closure['id']}", "reason": closure["reason"]}
+ for worker_id in ("first", "second")
+ for closure in CLOSURES
+ ]
+ if published:
+ parent_copy(tmp_path, expected)
+ originals = {path: path.read_bytes() for path in tmp_path.rglob("*.json")}
+ coverage = recover(tmp_path, workbench_api["saved_results"], workers)
+ assert sorted(coverage.get("resolvedDeferred", []), key=lambda row: row["id"]) == expected
+ assert all(path.read_bytes() == contents for path, contents in originals.items())
+
+
[email protected]("published", [False, True])
[email protected]("reopened_at", [19, 20, 21], ids=["older", "tied", "newer"])
+def test_worker_reopening_invalidates_its_copied_parent_closeout(
+ tmp_path, workbench_api, published, reopened_at
+):
+ worker = saved_worker(tmp_path, "worker")
+ output = Path(worker["artifact_dir"])
+ expected = [
+ {"id": f"worker-attempt-2-resolved-{closure['id']}", "reason": closure["reason"]}
+ for closure in CLOSURES
+ ]
+ if published:
+ parent_copy(tmp_path, expected)
+ reopened = {"id": "review-a", "reason": "New evidence requires another review."}
+ checkpoint = write_checkpoint(output / "checkpoints", saved_draft(SCAN_ID, deferred=[reopened]))
+ observed(checkpoint, reopened_at)
+ head = output / "checkpoint-head.json"
+ head.write_text(json.dumps({"checkpoint": checkpoint.name}))
+ observed(head, reopened_at)
+ coverage = recover(tmp_path, workbench_api["saved_results"], [worker])
+ assert sorted(coverage.get("resolvedDeferred", []), key=lambda row: row["id"]) == (
+ expected if reopened_at < 20 else expected[1:]
+ )
+ pending = [row for row in coverage["deferred"] if row.get("reason") == reopened["reason"]]
+ assert bool(pending) is (reopened_at >= 20)
+
+
+def test_archived_worker_closeout_keeps_its_original_attempt(tmp_path, workbench_api):
+ worker = saved_worker(tmp_path, "worker")
+ output = Path(worker["artifact_dir"])
+ archive = output.parent / "attempts" / "attempt-01"
+ archive.parent.mkdir(parents=True)
+ output.rename(archive)
+ output.mkdir()
+ result = output / "result.json"
+ result.write_text(json.dumps(saved_draft(SCAN_ID, complete=True)))
+ observed(result, 30)
+ coverage = recover(tmp_path, workbench_api["saved_results"], [worker])
+ assert sorted(coverage.get("resolvedDeferred", []), key=lambda row: row["id"]) == [
+ {"id": f"worker-attempt-1-resolved-{closure['id']}", "reason": closure["reason"]}
+ for closure in CLOSURES
+ ]
diff --git a/plugins/codex-security/tests/test_saved_deferred_identity.py b/plugins/codex-security/tests/test_saved_deferred_identity.py
index c316fad90..a644b6fc7 100644
--- a/plugins/codex-security/tests/test_saved_deferred_identity.py
+++ b/plugins/codex-security/tests/test_saved_deferred_identity.py
@@ -1,6 +1,7 @@
from __future__ import annotations
import copy
+import hashlib
import json
import os
import sys
@@ -706,9 +707,8 @@ def test_generic_surface_recovery_uses_resolved_candidate_identity(
head.write_text(json.dumps({"checkpoint": checkpoints[2 if rejected_here else 1].name}))
observed = 300 if rejected_here else 200
os.utime(head, ns=(observed, observed))
- rejection_worker_id = worker_id
if outcome == "other_worker":
- rejection_worker_id, other_result = accepted_standard_worker(
+ other_worker_id, other_result = accepted_standard_worker(
state, codex_home, scan_dir, scan_id, name="other-worker"
)
other_result.write_text(
@@ -719,31 +719,52 @@ def test_generic_surface_recovery_uses_resolved_candidate_identity(
first_coverage, recovered = cancel_and_preserve(
monkeypatch, saved_results, state, codex_home, scan_dir, scan_id
)
+ expected_surface_id = f"{worker_id}-attempt-1-surface-1"
+ expected_candidate = {
+ **candidate,
+ "id": f"{worker_id}-attempt-1-deferred-1",
+ "surfaceIds": [expected_surface_id],
+ "provenance": {"workerId": worker_id, "attempt": 1, "sourceId": candidate["id"]},
+ }
+ if "candidateId" in candidate:
+ expected_candidate["candidateId"] = (
+ f"{worker_id}-attempt-1-candidate-{hashlib.sha256(candidate['candidateId'].encode()).hexdigest()}"
+ )
+ expected_candidate["provenance"]["candidateId"] = candidate["candidateId"]
for coverage in (first_coverage, recovered):
- api_surfaces = [row for row in coverage["surfaces"] if row["id"] == f"{worker_id}/api"]
+ api_surfaces = [
+ row
+ for row in coverage["surfaces"]
+ if row.get("provenance") == {"workerId": worker_id, "attempt": 1, "sourceId": "api"}
+ ]
assert len(api_surfaces) == 1
+ actual_surface_id = api_surfaces[0]["id"]
+ assert isinstance(actual_surface_id, str)
expected_disposition = "no_issue_found" if rejected_here else "needs_follow_up"
assert api_surfaces[0]["disposition"] == expected_disposition
- assert api_surfaces[0]["workerId"] == worker_id
- assert not any(row["id"] == candidate["id"] for row in coverage["deferred"])
- assert not any(row["id"] in {"api", rejection["id"]} for row in coverage["surfaces"])
- assert not any(
- row["id"] in {generic["id"], f"{worker_id}/{generic['id']}"}
- for row in coverage["deferred"]
- )
- assert (
- any(row["id"] == f"{worker_id}/{candidate['id']}" for row in coverage["deferred"])
- ) is not rejected_here
+ assert api_surfaces[0] == {
+ **pending_surface,
+ "id": actual_surface_id,
+ "disposition": expected_disposition,
+ "provenance": {"workerId": worker_id, "attempt": 1, "sourceId": "api"},
+ }
+ assert not any(row["id"] == generic["id"] for row in coverage["deferred"])
+ assert any(row == expected_candidate for row in coverage["deferred"]) is not rejected_here
if outcome != "unresolved":
- assert {
+ owner = other_worker_id if outcome == "other_worker" else worker_id
+ index = 1 if outcome == "other_worker" else 2
+ expected_rejection = {
**rejection,
- "id": f"{rejection_worker_id}/{rejection['id']}",
- "workerId": rejection_worker_id,
- } in coverage["surfaces"]
- for row in coverage["deferred"]:
- if row["id"] == f"{worker_id}/{candidate['id']}":
- assert row["workerId"] == worker_id
- assert row["surfaceIds"] == [api_surfaces[0]["id"]]
+ "id": f"{owner}-attempt-1-surface-{index}",
+ "candidateId": f"{owner}-attempt-1-candidate-{hashlib.sha256(rejection['candidateId'].encode()).hexdigest()}",
+ "provenance": {
+ "workerId": owner,
+ "attempt": 1,
+ "sourceId": rejection["id"],
+ "candidateId": rejection["candidateId"],
+ },
+ }
+ assert expected_rejection in coverage["surfaces"]
assert recovered["surfaces"] == first_coverage["surfaces"]
assert recovered["deferred"] == first_coverage["deferred"]
@@ -873,20 +894,24 @@ def test_unnamed_observation_does_not_replace_saved_context(
first_coverage, replay = cancel_and_preserve(
monkeypatch, saved_results, state, codex_home, scan_dir, scan_id
)
- expected_output = (
- {**expected, "id": f"{worker_id}/{expected['id']}", "workerId": worker_id}
- if layout == "worker"
- else expected
- )
+ if layout == "worker" and observation == "explicit-update":
+ expected = {
+ **expected,
+ "id": f"{worker_id}-attempt-1-deferred-1",
+ "provenance": {"workerId": worker_id, "attempt": 1, "sourceId": broad["id"]},
+ }
+ elif layout == "worker":
+ expected = {**expected, "id": f"{worker_id}/{expected['id']}", "workerId": worker_id}
expected_raw = {**raw_row, "workerId": worker_id} if layout == "worker" else raw_row
for coverage in (first_coverage, replay):
pending = [row for row in coverage["deferred"] if row["id"] != "scan-stopped"]
- assert expected_output in pending
+ assert expected in pending
if layout == "worker":
- assert not any(row["id"] == broad["id"] for row in pending)
- assert all(row.get("workerId") == worker_id for row in pending)
- assert all(row["id"].startswith(f"{worker_id}/") for row in pending)
- unnamed = [row for row in pending if row.get("id") != expected_output["id"]]
+ assert all(
+ row.get("workerId", row.get("provenance", {}).get("workerId")) == worker_id
+ for row in pending
+ )
+ unnamed = [row for row in pending if row.get("id") != expected["id"]]
assert len(unnamed) == 1
assert {key: value for key, value in unnamed[0].items() if key != "id"} == expected_raw
assert "candidateId" not in unnamed[0]
diff --git a/plugins/codex-security/tests/test_workbench_checkpoint_heads.py b/plugins/codex-security/tests/test_workbench_checkpoint_heads.py
index 397057c06..1568c5600 100644
--- a/plugins/codex-security/tests/test_workbench_checkpoint_heads.py
+++ b/plugins/codex-security/tests/test_workbench_checkpoint_heads.py
@@ -79,6 +79,7 @@ def test_late_head_changes_require_explicit_recovery(
assert not any(path.endswith("checkpoint-head.json") for path in frozen)
assert not any(
row["id"] in {"review", f"{worker_id}/review"}
+ or row.get("provenance", {}).get("sourceId") == "review"
for row in json.loads((scan_dir / "coverage.json").read_text())["deferred"]
)
assert get_scan(state, scan_id)["scan"]["resultsRecoveryNeeded"] is False
@@ -92,15 +93,15 @@ def test_late_head_changes_require_explicit_recovery(
recovered = scan_command(state, "recover-scan-results", scan_id, environment=environment)
assert recovered["scan"]["resultsRecoveryNeeded"] is False
coverage = json.loads((scan_dir / "coverage.json").read_text())
- assert not any(row["id"] == "review" for row in coverage["deferred"])
- assert all(
- row.get("workerId") == worker_id
- for row in coverage["deferred"]
- if row["id"] != "scan-stopped"
- )
- assert any(row["id"] == f"{worker_id}/review" for row in coverage["deferred"]) is (
- selection == "reopened"
- )
+ expected = {"id": "review", "reason": "Review remains."}
+ if has_result:
+ expected.update(
+ id=f"{worker_id}-attempt-1-deferred-1",
+ provenance={"workerId": worker_id, "attempt": 1, "sourceId": "review"},
+ )
+ else:
+ expected.update(id=f"{worker_id}/review", workerId=worker_id)
+ assert (expected in coverage["deferred"]) is (selection == "reopened")
published = manifest_path.read_bytes()
scan_command(state, "recover-scan-results", scan_id, environment=environment)
assert manifest_path.read_bytes() == published
@@ -115,6 +116,43 @@ def checkpoint_scan():
return scan_id, pending, closed, binding
[email protected]("ancestry", [[], [{}], [[]], ["unknown", None, 7, False, {}, []]])
[email protected]("has_ancestor", [False, True])
+def test_frozen_parent_replay_ignores_nonreference_ancestry(
+ tmp_path: Path, checkpoint_scan, ancestry: list, has_ancestor: bool
+) -> None:
+ scan_id, pending, _, binding = checkpoint_scan
+ previous = []
+ if has_ancestor:
+ older = saved_draft(scan_id, complete=False)
+ older["previousParentCheckpoints"] = []
+ older["coverage"]["openQuestions"] = [{"question": "Earlier parent question."}]
+ checkpoint = write_checkpoint(tmp_path / "checkpoints", older)
+ previous.append(checkpoint.relative_to(tmp_path).as_posix())
+ os.utime(checkpoint, ns=(100, 100))
+ pending["previousParentCheckpoints"] = [*previous, *ancestry]
+ checkpoint = write_checkpoint(tmp_path / "checkpoints", pending)
+ os.utime(checkpoint, ns=(200, 200))
+ originals = {path: path.read_bytes() for path in (tmp_path / "checkpoints").iterdir()}
+ warnings = []
+ first = saved.merge_saved_results(
+ tmp_path, scan_id, binding, [], warnings, stopped=True, reason="interrupted"
+ )
+ assert warnings == []
+ assert {path.relative_to(tmp_path).as_posix() for path in originals} <= first[0]["scan"][
+ "preservedSources"
+ ].keys()
+ replay = replay_saved_results(saved, first, tmp_path, scan_id, binding)
+ repeated = replay_saved_results(saved, replay, tmp_path, scan_id, binding)
+ for result in (first, replay, repeated):
+ assert pending["coverage"]["deferred"][0] in result[2]["deferred"]
+ assert replay[2] == repeated[2]
+ assert replay[0]["scan"]["complete"] is True
+ if has_ancestor:
+ assert replay[2]["openQuestions"] == older["coverage"]["openQuestions"]
+ assert all(path.read_bytes() == original for path, original in originals.items())
+
+
@pytest.mark.parametrize("has_receipts", [False, True])
@pytest.mark.parametrize("has_canonical", [False, True])
def test_recovery_keeps_one_surface_with_optional_empty_receipts(
@@ -933,13 +971,16 @@ def test_parent_head_selection_matches_frozen_publication_retry(
json.loads((scan_dir / "findings.json").read_text()),
json.loads((scan_dir / "coverage.json").read_text()),
)
+ projected_child_work = {
+ **child_work,
+ "id": f"{worker_id}-attempt-1-deferred-1",
+ "provenance": {"workerId": worker_id, "attempt": 1, "sourceId": child_work["id"]},
+ }
for findings, coverage in (first[0], replay):
- assert not any(row["id"] == child_work["id"] for row in coverage["deferred"])
- assert {
- **child_work,
- "id": f"{worker_id}/{child_work['id']}",
- "workerId": worker_id,
- } in coverage["deferred"]
+ child_rows = [
+ row for row in coverage["deferred"] if row.get("reason") == child_work["reason"]
+ ]
+ assert child_rows == [projected_child_work]
if evidence == "deferred":
assert (parent_work in coverage["deferred"]) is (head_time <= 100)
else:
@@ -1136,6 +1177,76 @@ def test_older_attempt_selected_head_cannot_supersede_current_result(tmp_path):
assert result[2]["surfaces"] == [current]
[email protected]("prior_owner", ["shared", "worker"])
[email protected]("collision", [False, True])
+def test_generic_worker_closeout_preserves_receipt_owners(
+ tmp_path: Path, checkpoint_scan, prior_owner: str, collision: bool
+) -> None:
+ scan_id, _, _, binding = checkpoint_scan
+ output = tmp_path / "artifacts" / "deep_discovery" / "workers" / "worker" / "output"
+ archived = output.parent / "attempts" / "attempt-01"
+ output.mkdir(parents=True)
+ archived.mkdir(parents=True)
+ previous_ref = "artifacts/review.txt" if collision else "artifacts/previous.txt"
+ previous_path = (tmp_path if prior_owner == "shared" else archived) / previous_ref
+ previous_path.parent.mkdir(parents=True, exist_ok=True)
+ previous_path.write_text("Synthetic previous evidence.\n")
+ current_ref = "artifacts/review.txt" if collision else "artifacts/current.txt"
+ current_path = output / current_ref
+ current_path.parent.mkdir(parents=True, exist_ok=True)
+ current_path.write_text("Synthetic current evidence.\n")
+ surface = {
+ "id": "api",
+ "label": "API",
+ "disposition": "needs_follow_up",
+ "receiptRefs": [previous_ref],
+ }
+ pending = saved_draft(
+ scan_id,
+ surfaces=[surface],
+ deferred=[{"id": "review", "reason": "Review remains.", "surfaceIds": ["api"]}],
+ )
+ prior_checkpoint = write_checkpoint(archived / "checkpoints", pending)
+ os.utime(prior_checkpoint, ns=(100, 100))
+ (archived / "result.json").write_text(json.dumps(pending))
+ os.utime(archived / "result.json", ns=(100, 100))
+ closed = saved_draft(
+ scan_id,
+ surfaces=[{**surface, "disposition": "no_issue_found", "receiptRefs": [current_ref]}],
+ closures=[{"id": "review", "reason": "Review completed."}],
+ complete=True,
+ )
+ current_checkpoint = write_checkpoint(output / "checkpoints", closed)
+ os.utime(current_checkpoint, ns=(200, 200))
+ select(output, current_checkpoint, 300)
+ (output / "result.json").write_text(json.dumps(closed))
+ os.utime(output / "result.json", ns=(200, 200))
+ worker = {
+ **saved_discovery_worker(output, "worker", 2),
+ "status": "succeeded",
+ "result_manifest_path": str(output / "result.json"),
+ }
+ source_bytes = {path: path.read_bytes() for path in tmp_path.rglob("*.json")}
+ first = saved.merge_saved_results(
+ tmp_path, scan_id, binding, [worker], [], stopped=True, reason="interrupted"
+ )
+ expected = {
+ previous_path.relative_to(tmp_path).as_posix(),
+ current_path.relative_to(tmp_path).as_posix(),
+ }
+ for result in (first, replay_saved_results(saved, first, tmp_path, scan_id, binding, [worker])):
+ assert result is not None
+ recovered = [row for row in result[2]["surfaces"] if row["label"] == "API"]
+ assert len(recovered) == 1
+ assert recovered[0]["disposition"] == "no_issue_found"
+ assert set(recovered[0]["receiptRefs"]) == expected
+ assert {(tmp_path / ref).read_text() for ref in recovered[0]["receiptRefs"]} == {
+ "Synthetic previous evidence.\n",
+ "Synthetic current evidence.\n",
+ }
+ assert all(path.read_bytes() == contents for path, contents in source_bytes.items())
+
+
@pytest.mark.parametrize("complete", [False, True])
@pytest.mark.parametrize("abandoned_time", [100, 200, 300])
def test_accepted_head_survives_superseded_terminal_timestamps(
diff --git a/plugins/codex-security/tests/test_workbench_saved_source_order.py b/plugins/codex-security/tests/test_workbench_saved_source_order.py
index e4757095a..7a4fcfece 100644
--- a/plugins/codex-security/tests/test_workbench_saved_source_order.py
+++ b/plugins/codex-security/tests/test_workbench_saved_source_order.py
@@ -74,13 +74,18 @@ def test_failed_publication_keeps_order_after_identical_result_rewrite(
"standard-worker-thread",
)
coverage = json.loads((scan_dir / "coverage.json").read_text())
- assert not any(row["id"] == "review" for row in coverage["deferred"])
- assert all(
- row.get("workerId") == worker_id
- for row in coverage["deferred"]
- if row["id"] != "scan-stopped"
+ pending_rows = [row for row in coverage["deferred"] if row.get("reason") == "Review remains."]
+ assert pending_rows == (
+ [
+ {
+ **pending["coverage"]["deferred"][0],
+ "id": f"{worker_id}-attempt-1-deferred-1",
+ "provenance": {"workerId": worker_id, "attempt": 1, "sourceId": "review"},
+ }
+ ]
+ if latest_pending
+ else []
)
- assert any(row["id"] == f"{worker_id}/review" for row in coverage["deferred"]) is latest_pending
assert (
json.loads((scan_dir / "scan-manifest.json").read_text())["scan"]["preservedSources"]
== frozen
@@ -129,15 +134,16 @@ def test_head_capture_includes_checkpoint_published_after_enumeration(
)
assert len(published) == 1
coverage = json.loads((scan_dir / "coverage.json").read_text())
- expected_id = f"{worker_id}/review" if layout == "worker" else "review"
+ expected = pending["coverage"]["deferred"][0]
if layout == "worker":
- assert not any(row["id"] == "review" for row in coverage["deferred"])
- assert all(
- row.get("workerId") == worker_id
- for row in coverage["deferred"]
- if row["id"] != "scan-stopped"
- )
- assert any(row["id"] == expected_id for row in coverage["deferred"])
+ expected = {
+ **expected,
+ "id": f"{worker_id}-attempt-1-deferred-1",
+ "provenance": {"workerId": worker_id, "attempt": 1, "sourceId": "review"},
+ }
+ assert [row for row in coverage["deferred"] if row.get("reason") == "Review remains."] == [
+ expected
+ ]
sources = json.loads((scan_dir / "scan-manifest.json").read_text())["scan"]["preservedSources"]
assert published[0].relative_to(scan_dir).as_posix() in sources
assert any("/checkpoint-heads/" in f"/{path}" for path in sources)
diff --git a/plugins/codex-security/tests/test_workbench_standard_deep_results.py b/plugins/codex-security/tests/test_workbench_standard_deep_results.py
index ed945717f..b286f413c 100644
--- a/plugins/codex-security/tests/test_workbench_standard_deep_results.py
+++ b/plugins/codex-security/tests/test_workbench_standard_deep_results.py
@@ -18,6 +18,7 @@ from workbench_test_support import (
cancel_scan,
claim_deep_scan_dedup,
commit_deep_dedup,
+ empty_target_scan,
fail_deep_scan,
fail_scan,
finish_deep_scan,
@@ -770,6 +771,55 @@ def test_stopped_recovery_accepts_trailing_slash_scope(tmp_path: Path) -> None:
assert failed["findings"][0]["locations"][0]["path"] == "src/app.py"
+def test_canonical_finding_validation_survives_frozen_publication_retry(tmp_path: Path) -> None:
+ state_dir, target, scan_id, scan_dir = empty_target_scan(tmp_path)
+ write_completed_contract(scan_dir, scan_id, target)
+ findings_path = scan_dir / "findings.json"
+ findings = json.loads(findings_path.read_text())
+ finding = findings["findings"][0]
+ finding["severity"]["level"] = "informational"
+ stronger = copy.deepcopy(finding)
+ stronger["severity"]["level"] = "critical"
+ malformed_identity = copy.deepcopy(finding)
+ malformed_identity["identity"] = {"anchor": []}
+ findings["findings"].extend([stronger, malformed_identity])
+ findings_path.write_text(json.dumps(findings))
+ original = findings_path.read_bytes()
+
+ failed = run_workbench_with_fault(
+ tmp_path / "fail_canonical_publication.py",
+ state_dir,
+ tmp_path / "codex-home",
+ "def fail_publication(*args, **kwargs):\n"
+ " raise OSError('injected publication failure')\n"
+ "workbench_saved_results._write_prepared_scan_finalization = fail_publication\n",
+ "fail-scan",
+ "--scan-id",
+ scan_id,
+ "--message",
+ "Finding repair was interrupted.",
+ )
+ assert failed.returncode == 0, failed.stderr
+ assert findings_path.read_bytes() == original
+ assert get_scan(state_dir, scan_id)["scan"]["resultsRecoveryNeeded"] is True
+ frozen = {path: path.read_bytes() for path in (scan_dir / "checkpoints").glob("*.json")}
+ assert frozen
+
+ recovered = scan_command(state_dir, "recover-scan-results", scan_id)["scan"]
+
+ assert recovered["resultsRecoveryNeeded"] is False
+ assert recovered["findingCount"] == 1
+ finding = json.loads(findings_path.read_text())["findings"][0]
+ assert finding["severity"]["level"] == "critical"
+ assert any(
+ previous["severity"]["level"] == "informational"
+ for previous in finding["provenance"]["previousFindings"]
+ )
+ assert any("duplicate logical finding" in warning for warning in recovered["warnings"])
+ assert any("identity" in warning for warning in recovered["warnings"])
+ assert all(path.read_bytes() == content for path, content in frozen.items())
+
+
def test_canceled_scan_retries_failed_publication_from_frozen_sources(
tmp_path: Path,
) -> None:
@@ -2346,8 +2396,10 @@ def test_recovered_generic_closure_cannot_close_another_workers_same_id(
{
"id": worker_id,
"kind": "discovery",
+ "status": "running",
"artifact_dir": str(output),
"result_manifest_path": None,
+ "attempt": 1,
}
)
draft = saved_draft(
@@ -2393,7 +2445,16 @@ def test_recovered_generic_closure_cannot_close_another_workers_same_id(
row["reason"] for row in coverage["deferred"] if row["id"] != "scan-stopped"
} == expected
- assert not coverage.get("resolvedDeferred")
+ assert coverage.get("resolvedDeferred", []) == (
+ [
+ {
+ "id": "worker-one-attempt-1-resolved-shared-review",
+ "reason": "Review completed.",
+ }
+ ]
+ if closure_time == 200
+ else []
+ )
@pytest.mark.parametrize("canonical_time", [100, 200, 300])
@@ -2596,7 +2657,7 @@ def test_recovery_uses_frozen_worker_head_for_identical_reclosure(
os.utime(output / "result.json", ns=(300, 300))
workers = [saved_discovery_worker(output, "worker-one", 1 if layout == "current" else 2)]
if layout == "legacy_attempt_reopens":
- workers[0].pop("attempt")
+ workers[0]["attempt"] = 0
binding = saved_binding("deep_repository")
documents = workbench_saved_results.merge_saved_results(
scan_dir, scan_id, binding, workers, [], stopped=True, reason="interrupted"
diff --git a/plugins/codex-security/tests/test_workbench_stopped_worker_coverage.py b/plugins/codex-security/tests/test_workbench_stopped_worker_coverage.py
index 41f1fcb01..b7cbcb791 100644
--- a/plugins/codex-security/tests/test_workbench_stopped_worker_coverage.py
+++ b/plugins/codex-security/tests/test_workbench_stopped_worker_coverage.py
@@ -1,6 +1,7 @@
from __future__ import annotations
import json
+import uuid
from pathlib import Path
import pytest
@@ -15,12 +16,37 @@ from workbench_test_support import (
get_scan,
preserve_scan_results,
saved_draft,
+ upsert_deep_worker,
+ worker_paths,
)
+def coverage_worker(
+ state_dir: Path, codex_home: Path, scan_dir: Path, scan_id: str, name: str, accepted: bool
+) -> tuple[str, Path]:
+ if accepted:
+ return accepted_standard_worker(state_dir, codex_home, scan_dir, scan_id, name=name)
+ worker_id = str(uuid.uuid4())
+ prompt, artifacts, result = worker_paths(scan_dir, name)
+ upsert_deep_worker(
+ state_dir,
+ scan_id,
+ worker_id,
+ "discovery",
+ "running",
+ str(prompt),
+ str(artifacts),
+ "--attempt",
+ "1",
+ environment={"CODEX_HOME": str(codex_home)},
+ )
+ return worker_id, result
+
+
[email protected]("accepted", [False, True], ids=["running", "accepted"])
@pytest.mark.parametrize("termination", ["failed", "interrupted", "canceled"])
def test_stopped_deep_scan_projects_worker_coverage_without_parent_aggregate(
- tmp_path: Path, termination: str
+ tmp_path: Path, termination: str, accepted: bool
) -> None:
state_dir, codex_home, target, scan_dir, scan_id = deep_scan_fixture(tmp_path, workers=2)
environment = {"CODEX_HOME": str(codex_home)}
@@ -28,8 +54,8 @@ def test_stopped_deep_scan_projects_worker_coverage_without_parent_aggregate(
workers = {}
originals = {}
for name in ("first", "second"):
- worker_id, result = accepted_standard_worker(
- state_dir, codex_home, scan_dir, scan_id, name=name
+ worker_id, result = coverage_worker(
+ state_dir, codex_home, scan_dir, scan_id, name, accepted
)
receipt = result.parent / "artifacts/review.md"
receipt.parent.mkdir()
@@ -38,7 +64,7 @@ def test_stopped_deep_scan_projects_worker_coverage_without_parent_aggregate(
json.dumps(
saved_draft(
scan_id,
- complete=True,
+ complete=accepted,
findings=[finding] if name == "first" else [],
surfaces=[
{
@@ -73,14 +99,32 @@ def test_stopped_deep_scan_projects_worker_coverage_without_parent_aggregate(
coverage = json.loads((scan_dir / "coverage.json").read_text())
assert "completeness" not in coverage
assert {row["id"] for row in coverage["surfaces"]} == {
- f"{worker_id}/api" for worker_id, _ in workers.values()
+ f"{worker_id}-attempt-1-surface-1" if accepted else f"{worker_id}/api"
+ for worker_id, _ in workers.values()
}
for name, (worker_id, receipt) in workers.items():
- surface = next(row for row in coverage["surfaces"] if row["workerId"] == worker_id)
- task = next(row for row in coverage["deferred"] if row.get("workerId") == worker_id)
+ surface = next(row for row in coverage["surfaces"] if row["label"] == name)
+ task = next(
+ row for row in coverage["deferred"] if row["reason"] == f"Review {name} remains."
+ )
+ if accepted:
+ assert surface["provenance"] == {
+ "workerId": worker_id,
+ "attempt": 1,
+ "sourceId": "api",
+ }
+ assert task["provenance"] == {
+ "workerId": worker_id,
+ "attempt": 1,
+ "sourceId": "review",
+ }
+ else:
+ assert surface["workerId"] == task["workerId"] == worker_id
assert surface["disposition"] == "no_issue_found"
assert surface["receiptRefs"] == [receipt.relative_to(scan_dir).as_posix()]
- assert task["id"] == f"{worker_id}/review"
+ assert task["id"] == (
+ f"{worker_id}-attempt-1-deferred-1" if accepted else f"{worker_id}/review"
+ )
assert task["surfaceIds"] == [surface["id"]]
assert (scan_dir / surface["receiptRefs"][0]).read_text() == f"Synthetic {name} review.\n"
manifest = json.loads((scan_dir / "scan-manifest.json").read_text())
@@ -98,14 +142,19 @@ def test_stopped_deep_scan_projects_worker_coverage_without_parent_aggregate(
assert all(path.read_bytes() == content for path, content in originals.items())
-def test_stopped_worker_rejects_receipts_outside_its_artifacts_directory(tmp_path: Path) -> None:
[email protected]("accepted", [False, True], ids=["running", "accepted"])
+def test_stopped_worker_rejects_receipts_outside_its_artifacts_directory(
+ tmp_path: Path, accepted: bool
+) -> None:
state_dir, codex_home, _, scan_dir, scan_id = deep_scan_fixture(tmp_path)
- worker_id, result = accepted_standard_worker(state_dir, codex_home, scan_dir, scan_id)
+ worker_id, result = coverage_worker(
+ state_dir, codex_home, scan_dir, scan_id, "standard-worker", accepted
+ )
result.write_text(
json.dumps(
saved_draft(
scan_id,
- complete=True,
+ complete=accepted,
surfaces=[
{
"id": "api",
@@ -123,7 +172,11 @@ def test_stopped_worker_rejects_receipts_outside_its_artifacts_directory(tmp_pat
coverage_path = scan_dir / "coverage.json"
surface = json.loads(coverage_path.read_text())["surfaces"][0]
- assert surface["id"] == f"{worker_id}/api"
+ assert surface["id"] == (f"{worker_id}-attempt-1-surface-1" if accepted else f"{worker_id}/api")
+ if accepted:
+ assert surface["provenance"] == {"workerId": worker_id, "attempt": 1, "sourceId": "api"}
+ else:
+ assert surface["workerId"] == worker_id
assert surface["disposition"] == "needs_follow_up"
assert surface["receiptRefs"] == []
assert any(
diff --git a/plugins/codex-security/tests/test_worker_receipt_admission.py b/plugins/codex-security/tests/test_worker_receipt_admission.py
new file mode 100644
index 000000000..b02918f2b
--- /dev/null
+++ b/plugins/codex-security/tests/test_worker_receipt_admission.py
@@ -0,0 +1,261 @@
+from __future__ import annotations
+
+import copy
+import hashlib
+import json
+import os
+from argparse import Namespace
+from pathlib import Path
+
+import pytest
+from test_deep_scan_successful_publication import add_worker
+from test_deep_scan_successful_publication import publication_scan as publication_scan
+from test_partial_coverage_projection import stop_and_recover_projection
+from test_saved_deferred_identity import recover, save_worker
+from workbench_test_support import saved_draft, write_checkpoint
+
+
[email protected]("reordered", [False, True])
[email protected]("changed_notes", [False, True])
+def test_unselected_pending_checkpoint_links_to_accepted_worker_surface(
+ tmp_path: Path, reordered, changed_notes
+):
+ import workbench_saved_results as saved
+
+ surface = {
+ "id": "api",
+ "label": "API boundary",
+ "disposition": "needs_follow_up",
+ "receiptRefs": [],
+ }
+ surfaces = (
+ [{**surface, "id": "other", "label": "Other boundary"}, surface] if reordered else [surface]
+ )
+ accepted = saved_draft("identity-scan", surfaces=surfaces, complete=True)
+ worker = save_worker(tmp_path, saved, "reviewer", [], accepted)
+ output = tmp_path / "reviewer"
+ result_path = output / "result.json"
+ selected = write_checkpoint(output / "checkpoints", accepted)
+ head = output / "checkpoint-head.json"
+ head.write_text(json.dumps({"checkpoint": selected.name}))
+ for path in (selected, result_path, head):
+ os.utime(path, ns=(1_000_000_000, 1_000_000_000))
+ pending = copy.deepcopy(accepted)
+ pending["complete"] = False
+ pending["coverage"]["surfaces"].reverse()
+ if changed_notes:
+ for row in pending["coverage"]["surfaces"]:
+ row["notes"] = "Additional review context"
+ pending["coverage"]["deferred"] = [
+ {"id": "task", "reason": "Check boundary", "surfaceIds": ["api"]}
+ ]
+ unselected = write_checkpoint(output / "checkpoints", pending)
+ os.utime(unselected, ns=(2_000_000_000, 2_000_000_000))
+ originals = {path: path.read_bytes() for path in output.rglob("*.json")}
+ worker.update(status="succeeded", result_manifest_path=str(result_path))
+ recovered = recover(tmp_path, saved, [worker])
+ replay = recover(tmp_path, saved, [worker], recovered[0]["scan"]["preservedSources"])
+ assert recovered[2] == replay[2]
+ coverage = recovered[2]
+ reviewed = next(row for row in coverage["surfaces"] if row["label"] == "API boundary")
+ task = next(row for row in coverage["deferred"] if row["reason"] == "Check boundary")
+ assert task["surfaceIds"] == [reviewed["id"]]
+ assert all(path.read_bytes() == contents for path, contents in originals.items())
+
+
[email protected]("recorded", [False, True])
[email protected]("retry", [False, True])
[email protected]("archived", [False, True])
+def test_stopped_receipt_ownership_requires_a_matching_checkpoint(
+ workbench_api, workbench_db, publication_scan, monkeypatch, recorded, retry, archived
+):
+ scan = publication_scan()
+ initial = add_worker(workbench_db, scan)
+ worker_id = initial.parent.name
+ output = scan.scan_dir / "artifacts" / "deep_discovery" / "workers" / worker_id / "output"
+ output.mkdir(parents=True)
+ result = output / "result.json"
+ with workbench_db:
+ workbench_db.execute(
+ "UPDATE deep_scan_workers SET artifact_dir = ?, result_manifest_path = ? WHERE id = ?",
+ (str(output), str(result), worker_id),
+ )
+ ref = "artifacts/evidence.txt"
+ shared = scan.scan_dir / ref
+ shared.parent.mkdir(parents=True, exist_ok=True)
+ shared.write_text("Shared evidence\n")
+ draft = {
+ "scanId": scan.scan_id,
+ "complete": True,
+ "findings": [],
+ "coverage": {
+ "completeness": "complete",
+ "explicitExclusions": [],
+ "surfaces": [
+ {
+ "id": "review",
+ "label": "Reviewed route",
+ "disposition": "no_issue_found",
+ "receiptRefs": [ref],
+ "provenance": {"scanReceiptRefs": [ref]},
+ }
+ ],
+ "deferred": [],
+ },
+ }
+ result.write_text(json.dumps(draft))
+ if recorded:
+ write_checkpoint(result.parent / "checkpoints", draft)
+ local = result.parent / ref
+ local.parent.mkdir(parents=True, exist_ok=True)
+ local.write_text("Local evidence\n")
+ if archived:
+ archive = output.parent / "attempts" / "attempt-01"
+ archive.parent.mkdir()
+ output.rename(archive)
+ output.mkdir()
+ result = archive / "result.json"
+ local = archive / ref
+ with workbench_db:
+ workbench_db.execute(
+ "UPDATE deep_scan_workers SET status = 'running', attempt = 2, result_manifest_path = NULL WHERE id = ?",
+ (worker_id,),
+ )
+ original = result.read_bytes()
+ for name in ("scan-manifest.json", "findings.json", "coverage.json"):
+ (scan.scan_dir / name).unlink()
+ coverage = stop_and_recover_projection(workbench_api, workbench_db, scan, monkeypatch, retry)
+ assert len(coverage["surfaces"]) == 1
+ refs = coverage["surfaces"][0]["receiptRefs"]
+ selected = shared if recorded else local
+ assert refs == [selected.relative_to(scan.scan_dir).as_posix()]
+ assert (scan.scan_dir / refs[0]).read_text() == selected.read_text()
+ assert result.read_bytes() == original
+
+
+CANDIDATE_IDS = [
+ "candidate-\ud800",
+ "candidate-\udfff",
+ "candidate-\ud83d\ude00",
+ "candidate-\U0001f600",
+ "candidate-é",
+ "candidate-\\ud800",
+ "candidate-A\ud800B\udfffZ",
+]
+
+
+# Produced by the accepted JS draft parser and projectDiscoveryCoverage.
+JAVASCRIPT_CANDIDATE_HASHES = [
+ "48869f14875acce6da9a25e25db38d5a211fca5b07a2ac85d95dbeb7fbfad817",
+ "48869f14875acce6da9a25e25db38d5a211fca5b07a2ac85d95dbeb7fbfad817",
+ "a562be8af2b220690ce013d3fdce2d1db00a5cd0fffb191389023b836c65d43c",
+ "a562be8af2b220690ce013d3fdce2d1db00a5cd0fffb191389023b836c65d43c",
+ "e58295b9114eb779fb2c91d375fe1c103056bfe760954a8fbf7954a84c2cbb6a",
+ "f6d3552df9eaff1e1ce7185aa6efb48eff97d5a74a39a44c5a4b7ff55fbe43c4",
+ "d80ce56ca1e18843faee20969b09a04c1f32969a104b9ccf050f0a345eeb3912",
+]
+
+
[email protected]("index", range(len(CANDIDATE_IDS)))
+def test_saved_candidate_projection_matches_javascript_utf8(tmp_path: Path, index: int):
+ import workbench_saved_results as saved
+ from workbench_validation import javascript_utf8
+
+ candidate = CANDIDATE_IDS[index]
+ digest = JAVASCRIPT_CANDIDATE_HASHES[index]
+ assert hashlib.sha256(javascript_utf8(candidate)).hexdigest() == digest
+ draft = saved_draft(
+ "identity-scan",
+ complete=True,
+ surfaces=[
+ {
+ "id": "review",
+ "label": "Reviewed route",
+ "candidateId": candidate,
+ "disposition": "no_issue_found",
+ "receiptRefs": [],
+ }
+ ],
+ )
+ worker = save_worker(tmp_path, saved, "reviewer", [], draft)
+ worker["status"] = "succeeded"
+ worker["result_manifest_path"] = str(tmp_path / "reviewer" / "result.json")
+ result_path = Path(worker["result_manifest_path"])
+ original = result_path.read_bytes()
+ result = recover(tmp_path, saved, [worker])
+ surface = result[2]["surfaces"][0]
+ assert surface["candidateId"] == f"reviewer-attempt-1-candidate-{digest}"
+ assert surface["provenance"]["candidateId"] == json.loads(json.dumps(candidate))
+ assert recover(tmp_path, saved, [worker], result[0]["scan"]["preservedSources"])[2] == result[2]
+ assert result_path.read_bytes() == original
+
+
[email protected]("outcome", ["failed", "canceled"])
[email protected]("changed_checkout", [False, True])
+def test_legacy_file_scope_preserves_shared_receipts_when_stopped(
+ workbench_api, workbench_db, publication_scan, outcome, changed_checkout
+):
+ scope = "subdir/extract.py"
+ scan = publication_scan(paths=[scope])
+ db = workbench_api["_WORKBENCH_DB_CONTEXT"]
+ registered = db.require_scan(workbench_db, scan.scan_id)
+ assert db.requested_finding_scope(registered) == [(scope, True)]
+ legacy_path = (
+ Path(__file__).parents[1] / "mcp-app/tests/fixtures/legacy-scoped-worker-receipts.json"
+ )
+ legacy = json.loads(legacy_path.read_text())
+ for draft in legacy.values():
+ draft["scanId"] = scan.scan_id
+ draft["findings"][0]["locations"][0]["path"] = scope
+ legacy["checkpoint"]["findings"][1]["locations"][0]["path"] = f"{scope}/generated"
+ initial = add_worker(workbench_db, scan)
+ worker_id = initial.parent.name
+ output = scan.scan_dir / "artifacts" / "deep_discovery" / "workers" / worker_id / "output"
+ output.mkdir(parents=True)
+ result = output / "result.json"
+ with workbench_db:
+ workbench_db.execute(
+ "UPDATE deep_scan_workers SET artifact_dir = ?, result_manifest_path = ? WHERE id = ?",
+ (str(output), str(result), worker_id),
+ )
+ result.write_text(json.dumps(legacy["result"]))
+ checkpoint = write_checkpoint(result.parent / "checkpoints", legacy["checkpoint"])
+ os.utime(checkpoint, ns=(1_000_000_000, 1_000_000_000))
+ os.utime(result, ns=(2_000_000_000, 2_000_000_000))
+ ref = "artifacts/evidence.txt"
+ shared, local = scan.scan_dir / ref, result.parent / ref
+ for path, contents in ((shared, "Shared evidence\n"), (local, "Unrelated local evidence\n")):
+ path.parent.mkdir(parents=True, exist_ok=True)
+ path.write_text(contents)
+ original = {path: path.read_bytes() for path in (result, checkpoint, shared, local)}
+ if changed_checkout:
+ target = Path(registered["target_path"]) / scope
+ target.unlink()
+ target.mkdir()
+ for name in ("scan-manifest.json", "findings.json", "coverage.json"):
+ (scan.scan_dir / name).unlink()
+ saved = workbench_api["saved_results"]
+ args = Namespace(
+ scan_id=scan.scan_id,
+ claim_token=None,
+ thread_id=None,
+ cost_json=None,
+ message="Synthetic stopped review.",
+ coordinator_generation=None,
+ )
+ (saved.fail_scan if outcome == "failed" else saved.cancel_scan)(db, workbench_db, args)
+ coverage_path = scan.scan_dir / "coverage.json"
+ coverage = json.loads(coverage_path.read_text())
+ assert len(coverage["surfaces"]) == 1
+ assert coverage["surfaces"][0]["receiptRefs"] == [ref]
+ assert (
+ scan.scan_dir / coverage["surfaces"][0]["receiptRefs"][0]
+ ).read_text() == "Shared evidence\n"
+ manifest = json.loads((scan.scan_dir / "scan-manifest.json").read_text())
+ assert manifest["scan"]["status"] == outcome
+ assert "filePaths" not in manifest["scan"]["scope"]
+ assert "filePaths" not in coverage
+ published = coverage_path.read_bytes()
+ saved.preserve_scan_results(db, workbench_db, args)
+ assert coverage_path.read_bytes() == published
+ assert {path: path.read_bytes() for path in original} == original
diff --git a/plugins/codex-security/tests/workbench_test_support.py b/plugins/codex-security/tests/workbench_test_support.py
index 421dc01b7..210e92460 100644
--- a/plugins/codex-security/tests/workbench_test_support.py
+++ b/plugins/codex-security/tests/workbench_test_support.py
@@ -98,6 +98,8 @@ def saved_discovery_worker(output: Path, worker_id: str = "worker", attempt: int
return {
"id": worker_id,
"kind": "discovery",
+ "status": "running",
+ "merge_state": "buffered",
"artifact_dir": str(output),
"result_manifest_path": None,
"attempt": attempt,
diff --git a/sdk/typescript/tests-ts/plugin-root.ts b/sdk/typescript/tests-ts/plugin-root.ts
index cce6d818d..06341ad48 100644
--- a/sdk/typescript/tests-ts/plugin-root.ts
+++ b/sdk/typescript/tests-ts/plugin-root.ts
@@ -1,5 +1,5 @@
import { existsSync } from "node:fs";
-import { chmod, cp, readFile, writeFile } from "node:fs/promises";
+import { chmod, cp, readdir, readFile, writeFile } from "node:fs/promises";
import { join } from "node:path";
import { fileURLToPath } from "node:url";
import { brotliDecompressSync } from "node:zlib";
@@ -18,13 +18,18 @@ export const INTEGRATION_TARGET = hasMonorepoSdk
let bundledRuntime: Promise<string> | undefined;
export function loadBundledRuntime(): Promise<string> {
- return (bundledRuntime ??= Promise.all(
- ["000", "001"].map((part) =>
- readFile(new URL(`mcp/server.mjs.br.part-${part}`, bundledPlugin)),
- ),
- ).then((parts) =>
- brotliDecompressSync(Buffer.concat(parts)).toString("utf8"),
- ));
+ return (bundledRuntime ??= readdir(new URL("mcp/", bundledPlugin))
+ .then((names) =>
+ Promise.all(
+ names
+ .filter((name) => name.startsWith("server.mjs.br.part-"))
+ .sort()
+ .map((name) => readFile(new URL(`mcp/${name}`, bundledPlugin))),
+ ),
+ )
+ .then((parts) =>
+ brotliDecompressSync(Buffer.concat(parts)).toString("utf8"),
+ ));
}
export const copyCompletedScanFixture = (destination: string) =>