Skip to content

OpenAI's Codex Security CLI and TypeScript SDK for finding, validating, and fixing security vulnerabilities. npm: https://www.npmjs.com/package/@openai/codex-security

fix(imports): preserve boxed numeric evidence across retries (#2224)

Michael D'Angelo7c52fd3c66d6
diff --git a/docs/scanner-imports/schema-guide.md b/docs/scanner-imports/schema-guide.md
index 44d00991a..81f494e11 100644
--- a/docs/scanner-imports/schema-guide.md
+++ b/docs/scanner-imports/schema-guide.md
@@ -47,6 +47,17 @@ defines the fields the reader can populate.
| `details` | Optional context for `sast`, `secret`, `iac` or `advisory` findings. For GHSA, `details.advisory.source` distinguishes repository and global disclosures. |
| `source_data` | The vendor fields retained for this record. Keep the original export when you need scan-wide diagnostics or inventory. |
+Use finite JSON numbers in evidence and report metadata. Supply integers outside
+JavaScript’s safe range as strings to preserve their value. In SDK inputs, use
+primitive numbers in plain data fields for predictable JSON exports. Supply
+precise decimals as strings if converting a `JSON.rawJSON()` number to a
+JavaScript number would round its value.
+
+When you supply normalized findings to the SDK, new imports use their JSON
+representation. For example, `new Number(7)` exports as `7`. If its `toJSON()`
+returns `9`, the new import uses `9`. The GitHub code-scanning and advisory
+readers also apply the Cloud size limit to the evidence they return.
+
## Operate the two destinations
**Local.** Run `codex-security findings imported PATH --format json` with the
@@ -92,7 +103,16 @@ With the SDK, repeat the preparation call that created the submission, using
the same options. For `prepareExternalPublication()`, supply the original
export file. For `prepareNormalizedExternalPublication()`, supply the original
`ExternalFindings`. Check `preview.resumed` before calling `publish()`.
-Changing the findings prepares a new submission.
+Use the same SDK version throughout a submission when possible.
+
+After an SDK upgrade, retry with the original input. An older SDK could approve
+different bytes from those its JSON export contained. When the SDK verifies
+the old selection and request bytes, it resumes with the original request IDs
+and skips batches that already have receipts. If it cannot verify the original
+request, it preserves the checkpoint and stops before uploading. A saved JSON
+export can lose information from the original SDK input. If a getter makes it
+impossible to identify which findings were approved, the SDK can also stop a
+new submission for the same source and destination.
See [results and exit codes](../../sdk/typescript/docs/cli.md#results-and-exit-codes)
for cumulative counts and interrupted readback.
diff --git a/sdk/typescript/scripts/package-dist-files.mjs b/sdk/typescript/scripts/package-dist-files.mjs
index 57c085db5..0afbb6b64 100644
--- a/sdk/typescript/scripts/package-dist-files.mjs
+++ b/sdk/typescript/scripts/package-dist-files.mjs
@@ -27,6 +27,7 @@ const distModules = [
"external-findings-local",
"external-findings-import",
"external-findings-read",
+ "reader-selection",
"scanner-import",
"inspector-findings",
"snyk-findings",
diff --git a/sdk/typescript/src/external-findings-local.ts b/sdk/typescript/src/external-findings-local.ts
index 9edb1e3d3..6c3bcf750 100644
--- a/sdk/typescript/src/external-findings-local.ts
+++ b/sdk/typescript/src/external-findings-local.ts
@@ -1,19 +1,20 @@
-import { isNumberObject } from "node:util/types";
import { CodexSecurityError } from "./errors.js";
import {
forwardExternalImportSetting,
requireExternalImports,
} from "./external-imports-feature.js";
import {
+ cloneImportPreview,
snapshotImportOptions,
+ snapshotNormalizedFindings,
snapshotSourceRepositories,
type ExternalFindings,
type ExternalSourceRepository,
} from "./external-findings.js";
import {
cloudRepositoryUrlKey,
+ requireSafeJsonInteger,
repositoryUrlKey,
- validateExternalReport,
} from "./external-import-contract.js";
import type {
ExternalFindingEvidence,
@@ -125,6 +126,14 @@ function requireIdentity(
);
}
+function checkStoredNumber(value: number): void {
+ if (!Number.isFinite(value))
+ throw new TypeError(
+ "Stored finding JSON cannot contain non-finite numbers.",
+ );
+ requireSafeJsonInteger(value, "Stored finding JSON");
+}
+
const gitTransportPorts: Readonly<Record<string, string>> = {
"http:": "80",
"https:": "443",
@@ -650,8 +659,7 @@ export async function prepareLocalImport(
fetchRemotes?: readonly ConfiguredFetchRemote[],
): Promise<PreparedLocalExternalImport> {
requireIdentity(options.sourceKey, "Source key");
- const current = structuredClone(input);
- validateExternalReport(current.report);
+ const current = snapshotNormalizedFindings(input, checkStoredNumber);
let dependabotSource: string | undefined;
const identities = new Set<string>();
for (const finding of current.findings) {
@@ -903,26 +911,19 @@ export async function prepareLocalImport(
);
dependencies.signal?.throwIfAborted();
const preview = {
- ...structuredClone(current),
+ ...cloneImportPreview(current),
repository,
source: { provider: options.provider, source_key: options.sourceKey },
};
- const payload = JSON.stringify(
- {
- repository,
- provider: options.provider,
- sourceKey: options.sourceKey,
- findings: current.findings,
- },
- (_key, value: unknown) => {
- if (isNumberObject(value)) value = +value;
- if (typeof value === "number" && !Number.isFinite(value))
- throw new TypeError(
- "Stored finding JSON cannot contain non-finite numbers.",
- );
- return value;
- },
- );
+ const payload = JSON.stringify({
+ repository,
+ provider: options.provider,
+ sourceKey: options.sourceKey,
+ findings: current.findings.map(({ source_finding_id, evidence }) => ({
+ source_finding_id,
+ evidence,
+ })),
+ });
return {
matchesRepositoryRemote,
preview,
@@ -966,7 +967,7 @@ export async function prepareLocalImport(
read: current.read,
...(current.report === undefined
? {}
- : { report: structuredClone(current.report) }),
+ : snapshotNormalizedFindings({ report: current.report })),
ready: current.findings.length,
excluded: structuredClone(current.excluded),
findings: result["findings"] as unknown as StoredExternalFinding[],
diff --git a/sdk/typescript/src/external-findings-publish.ts b/sdk/typescript/src/external-findings-publish.ts
index 7018606f3..78498d6e2 100644
--- a/sdk/typescript/src/external-findings-publish.ts
+++ b/sdk/typescript/src/external-findings-publish.ts
@@ -12,13 +12,14 @@ import {
import { createRequire } from "node:module";
import { join } from "node:path";
import { setTimeout as delay } from "node:timers/promises";
+import { isBoxedPrimitive, isNativeError, isProxy } from "node:util/types";
import { readCloudCredentials } from "./cloud-publish.js";
import {
cloudBaseUrl,
cloudFindingsUrl,
DEFAULT_CLOUD_BASE_URL,
} from "./cloud-endpoint.js";
-import { CodexSecurityError } from "./errors.js";
+import { CodexSecurityError, errorMessage } from "./errors.js";
import { requireExternalImports } from "./external-imports-feature.js";
import {
codexSecurityStateDirectory,
@@ -30,10 +31,15 @@ import {
readExternalFindings,
type ExternalFindingsReaderOptions,
} from "./external-findings-read.js";
-import { canonicalJson } from "./record.js";
+import {
+ canSnapshotWithoutGetters,
+ canonicalJson,
+ isRecord,
+} from "./record.js";
import { notify } from "./value.js";
import { wizCsvRepositoryUrl } from "./wiz-csv.js";
import { wizFindingRepository } from "./wiz-findings.js";
+import { previousReaderSelection } from "./reader-selection.js";
import { dependabotRepository } from "./github-dependabot-findings.js";
import { snykRepositoryUrls } from "./snyk-findings.js";
import { endorRepositoryUrls } from "./endor-findings.js";
@@ -46,13 +52,18 @@ import { inspectorReportRepository } from "./inspector-findings.js";
import { sonarRepositoryUrls } from "./sonar-findings.js";
import {
snapshotImportOptions,
+ cloneImportPreview,
+ nativeErrorStackGetter,
+ snapshotNormalizedFindings,
type ExternalFindings,
type ExternalPublicationInput,
} from "./external-findings.js";
import {
cloudRepositoryUrlKey,
+ CloudEvidenceDepthError,
validateImportRequest,
validateExternalEvidence,
+ validatePreviousCloudEvidence,
validateExternalReport,
validateImportReceipt,
validateRepositories,
@@ -140,7 +151,17 @@ interface SavedSubmission {
accountId: string;
requests: FindingImportRequest[];
receipts?: FindingImportReceipt[];
- inputProjection?: { inputDigest: string; savedEvidenceDigest: string };
+ inputProjection?: {
+ identityFormat?: 2;
+ inputDigest: string;
+ savedEvidenceDigest: string;
+ };
+ // A verified older approval stays at its original deployment-bound filename.
+ wireKey?: string;
+ // Only a verified released approval may preserve a native value different from fresh JSON.
+ legacyApproval?: true;
+ // The older canonical file was checked under its lock for a second approval.
+ canonicalChecked?: true;
inspectorProjectMapRequired?: true;
}
@@ -178,6 +199,16 @@ async function readInBatches<T, U>(
return output;
}
+// JSON.stringify serializes boxed primitives after the replacer. Let it
+// produce their actual wire values, including false and strings.
+function wireJson(value: unknown): string {
+ return canonicalJson(value, undefined, isBoxedPrimitive);
+}
+
+function jsonDigest(value: unknown, serialize = wireJson): string {
+ return hash("sha256", serialize(value));
+}
+
function sameSubmission(
left: SavedSubmission,
right: SavedSubmission,
@@ -185,18 +216,63 @@ function sameSubmission(
return (
left.accountId === right.accountId &&
left.inspectorProjectMapRequired === right.inspectorProjectMapRequired &&
- canonicalJson(left.requests) === canonicalJson(right.requests)
+ wireJson(left.requests) === wireJson(right.requests)
);
}
+function wireFinding({
+ source_finding_id,
+ evidence,
+}: ExternalFindings["findings"][number]) {
+ return { source_finding_id, evidence };
+}
+
function submissionFindings(
requests: readonly FindingImportRequest[],
): ExternalFindings["findings"] {
- return requests.flatMap((batch) =>
- batch.items.map(({ source_finding_id, evidence }) => ({
- source_finding_id,
- evidence,
- })),
+ return requests.flatMap((batch) => batch.items.map(wireFinding));
+}
+
+function sameLegacyJson(
+ saved: unknown,
+ current: unknown,
+ equivalent: (saved: unknown, current: unknown) => boolean,
+): boolean {
+ if (saved === current || equivalent(saved, current)) return true;
+ if (Array.isArray(saved) && Array.isArray(current))
+ return (
+ saved.length === current.length &&
+ saved.every((child, i) => sameLegacyJson(child, current[i], equivalent))
+ );
+ if (!isRecord(saved) || !isRecord(current)) return false;
+ return (
+ Object.keys(current).every((key) => Object.hasOwn(saved, key)) &&
+ Object.keys(saved).every((key) =>
+ sameLegacyJson(saved[key], current[key], equivalent),
+ )
+ );
+}
+
+// A copied reader result can retain the JSON from Buffer.toJSON but lose
+// the native Buffer that the previous reader cloned into a Uint8Array.
+function previousBufferJson(saved: unknown, current: unknown): boolean {
+ if (
+ !isRecord(current) ||
+ current["type"] !== "Buffer" ||
+ !Array.isArray(current["data"])
+ )
+ return false;
+ return (
+ isRecord(saved) &&
+ Object.keys(current).length === 2 &&
+ Object.keys(saved).length === current["data"].length &&
+ current["data"].every(
+ (byte, index) =>
+ Number.isInteger(byte) &&
+ byte >= 0 &&
+ byte <= 255 &&
+ saved[index] === byte,
+ )
);
}
@@ -224,7 +300,7 @@ function reportForJson(report: ExternalFindings["report"]) {
const errors = report?.["errors"];
if (!Array.isArray(errors)) return report;
const json = JSON.stringify(errors, (_key, entry: unknown) =>
- entry instanceof Error ? { message: entry.message } : entry,
+ entry instanceof Error ? { ...entry, message: entry.message } : entry,
);
return { ...report, errors: JSON.parse(json) as unknown[] };
}
@@ -278,8 +354,8 @@ function validatePublicationReceipt(
const batchItems = new Map(batch.items.map((item) => [item.client_id, item]));
if (
receipt.id !== batch.request_id ||
- canonicalJson(receipt.repository) !== canonicalJson(batch.repository) ||
- canonicalJson(receipt.source) !== canonicalJson(batch.source) ||
+ wireJson(receipt.repository) !== wireJson(batch.repository) ||
+ wireJson(receipt.source) !== wireJson(batch.source) ||
receipt.item_count !== batch.items.length ||
receipt.results.length !== batch.items.length ||
new Set(receipt.results.map((item) => item.client_id)).size !==
@@ -314,16 +390,16 @@ function validatePublicationReceipt(
"Cloud returned an incomplete saved finding.",
);
}
- if (canonicalJson(receipt.counts) !== canonicalJson(actual))
+ if (wireJson(receipt.counts) !== wireJson(actual))
throw new CodexSecurityError(
"Cloud returned inconsistent publication counts.",
);
return receipt;
}
-async function withImportLock<T>(
+async function withImportLocks<T>(
state: string,
- key: string,
+ keys: string[],
signal: AbortSignal | undefined,
operation: () => Promise<T>,
): Promise<T> {
@@ -331,40 +407,53 @@ async function withImportLock<T>(
await createWindowsOutputDirectory(state, true);
else await mkdir(state, { recursive: true, mode: 0o700 });
await validatePreparedOutputDir(state, undefined, true);
- const path = join(state, `${key}.lock.sqlite`);
- const metadata = await lstat(path).catch((error: NodeJS.ErrnoException) => {
- if (error.code === "ENOENT") return undefined;
- throw error;
- });
- if (metadata && !metadata.isFile())
- throw new CodexSecurityError("The publication lock is not a regular file.");
const require = createRequire(import.meta.url);
const Database = (
process.versions["bun"]
? require("bun:sqlite").Database
: require("node:sqlite").DatabaseSync
) as new (path: string) => { exec(sql: string): void; close(): void };
- const guard = new Database(path);
+ const guards: InstanceType<typeof Database>[] = [];
try {
- if (process.platform !== "win32") chmodSync(path, 0o600);
- guard.exec("PRAGMA busy_timeout = 0");
+ for (const key of [...new Set(keys)].sort()) {
+ const path = join(state, `${key}.lock.sqlite`);
+ const metadata = await lstat(path).catch(
+ (error: NodeJS.ErrnoException) => {
+ if (error.code === "ENOENT") return undefined;
+ throw error;
+ },
+ );
+ if (metadata && !metadata.isFile())
+ throw new CodexSecurityError(
+ "The publication lock is not a regular file.",
+ );
+ const guard = new Database(path);
+ guards.push(guard);
+ if (process.platform !== "win32") chmodSync(path, 0o600);
+ guard.exec("PRAGMA busy_timeout = 0");
+ }
for (;;) {
signal?.throwIfAborted();
+ let acquired = 0;
try {
- // Keep this inode across invocations. SQLite releases the transaction
- // on process exit, so a paused publisher cannot lose its lock to a lease.
- guard.exec("BEGIN EXCLUSIVE");
+ // SQLite releases these transactions on process exit.
+ for (const guard of guards) {
+ guard.exec("BEGIN EXCLUSIVE");
+ acquired++;
+ }
break;
} catch (error) {
const sqliteError = error as { code?: string; errcode?: number };
if (sqliteError.errcode !== 5 && sqliteError.code !== "SQLITE_BUSY")
throw error;
+ // Older SDKs can acquire historical keys in a different order.
+ for (const guard of guards.slice(0, acquired)) guard.exec("ROLLBACK");
await delay(50, undefined, { signal });
}
}
return await operation();
} finally {
- guard.close();
+ for (const guard of guards) guard.close();
}
}
@@ -398,51 +487,333 @@ export async function prepareNormalizedExternalPublication(
);
}
-function normalizedPublicationInput(
+interface PublicationSelection extends ExternalPublicationInput {
+ original?: {
+ parsed: ExternalFindings;
+ identity: unknown;
+ wire: string;
+ hasBoxedScalars: boolean;
+ };
+
+ potentialIds?: (string | undefined)[] | null;
+ uncertainFindingIds?: string[];
+ unprovenOccurrenceCount?: number;
+ knownFindings?: ExternalFindings["findings"];
+ unprovenIdPrefixes?: string[];
+ freshError?: Error;
+}
+
+function ownData(value: unknown, key: string) {
+ return isRecord(value) && !isProxy(value)
+ ? (Object.getOwnPropertyDescriptor(value, key)?.value as unknown)
+ : undefined;
+}
+
+// When native capture would invoke caller code, only the original data
+// properties can identify the earlier selection. Unknown IDs cannot rule it out.
+function previousFindingIds(
input: ExternalFindings,
-): ExternalPublicationInput {
- // Validation adds Cloud defaults; leave the caller's objects unchanged.
- const { report, ...parsed } = input;
- const current: ExternalFindings = structuredClone(parsed);
- if (report !== undefined) {
- current.report = structuredClone(report);
- validateExternalReport(current.report);
+): (string | undefined)[] | null {
+ const rows = ownData(input, "findings");
+ if (!Array.isArray(rows) || isProxy(rows)) return null;
+ return Array.from({ length: rows.length }, (_, index) => {
+ const row: unknown = Object.getOwnPropertyDescriptor(rows, index)?.value;
+ const id = ownData(row, "source_finding_id");
+ return typeof id === "string" ? id : undefined;
+ });
+}
+
+function withDataProperty<T extends object>(
+ record: T,
+ key: string,
+ value: unknown,
+): T {
+ const fields = Object.getOwnPropertyDescriptors(record);
+ return Object.create(Object.getPrototypeOf(record) as object | null, {
+ ...fields,
+ [key]: { ...fields[key], value },
+ }) as T;
+}
+
+// Native cloning discarded enumerable diagnostic fields. Recover its message
+// without reading the caller's stack getter or changing their report.
+const nativeDiagnosticPrototypes = new Set<unknown>([
+ Error.prototype,
+ EvalError.prototype,
+ RangeError.prototype,
+ ReferenceError.prototype,
+ SyntaxError.prototype,
+ TypeError.prototype,
+ URIError.prototype,
+]);
+
+function previousDiagnosticMessage(diagnostic: Error): string | undefined {
+ const stack = Object.getOwnPropertyDescriptor(diagnostic, "stack");
+ const message = ownData(diagnostic, "message");
+ const cause = Object.getOwnPropertyDescriptor(diagnostic, "cause");
+ const name = Object.getOwnPropertyDescriptor(diagnostic, "name");
+ if (
+ !nativeDiagnosticPrototypes.has(Object.getPrototypeOf(diagnostic)) ||
+ typeof message !== "string" ||
+ (stack?.get && stack.get !== nativeErrorStackGetter) ||
+ (name && typeof name.value !== "string") ||
+ (cause &&
+ (!Object.hasOwn(cause, "value") ||
+ !canSnapshotWithoutGetters(cause.value)))
+ )
+ return;
+ return message;
+}
+
+function previousDiagnosticInput(
+ input: ExternalFindings,
+ nativeDiagnostics: Map<number, Error>,
+): ExternalFindings {
+ const report = ownData(input, "report");
+ const errors = ownData(report, "errors");
+ if (!isRecord(report) || !Array.isArray(errors) || isProxy(errors))
+ return input;
+ let entries: PropertyDescriptorMap | undefined;
+ for (let index = 0; index < errors.length; index++) {
+ const descriptor = Object.getOwnPropertyDescriptor(errors, index);
+ const diagnostic: unknown = descriptor?.value;
+ if (!descriptor || !isNativeError(diagnostic)) continue;
+ const message = previousDiagnosticMessage(diagnostic);
+ if (message === undefined) return input;
+ nativeDiagnostics.set(index, diagnostic);
+ (entries ??= Object.getOwnPropertyDescriptors(errors as object))[index] = {
+ ...descriptor,
+ value: { message },
+ };
}
- // structuredClone preserves aliases. Detach every raw record before adding
- // typed defaults, including aliases shared with another finding's evidence.
- for (const finding of current.findings)
- if (finding.evidence.source_data !== undefined)
- finding.evidence.source_data = structuredClone(
- finding.evidence.source_data,
- );
- const identities = new Set<string>();
- for (const finding of current.findings) {
+ if (!entries) return input;
+ const capturedErrors = Object.defineProperties([], entries) as unknown[];
+ const capturedReport = withDataProperty(report, "errors", capturedErrors);
+ return withDataProperty(input, "report", capturedReport);
+}
+
+// Capture before JSON hooks can change their source. If native capture would
+// invoke caller code too, the original bytes cannot be observed independently.
+function previousNormalizedFindings(
+ input: ExternalFindings,
+ nativeDiagnostics: Map<number, Error>,
+ observe: (value: object) => void,
+): ExternalFindings | undefined {
+ input = previousDiagnosticInput(input, nativeDiagnostics);
+ if (!canSnapshotWithoutGetters(input, observe)) return;
+ // Object spread skips a hidden report, but explicitly reading it could run a getter.
+ const reportField = Object.getOwnPropertyDescriptor(input, "report");
+ if (reportField && !Object.hasOwn(reportField, "value")) return;
+ const { report, ...parsed } = input;
+ const snapshot: ExternalFindings = structuredClone(parsed);
+ if (report !== undefined) snapshot.report = structuredClone(report);
+ return snapshot;
+}
+
+function publicationIdentity(parsed: ExternalFindings) {
+ return parsed.report === undefined
+ ? parsed.findings
+ : { findings: parsed.findings, report: reportForJson(parsed.report) };
+}
+
+function validatePublicationFindings(
+ parsed: ExternalFindings,
+ validateEvidence: typeof validateExternalEvidence,
+) {
+ validateExternalReport(parsed.report);
+ const ids = new Set<string>();
+ for (const finding of parsed.findings) {
const id = finding.source_finding_id;
if (!id.trim() || id.includes("\0") || Buffer.byteLength(id) > 512)
throw new CodexSecurityError(
"Source finding id must contain 1–512 UTF-8 bytes and no NUL characters.",
);
- if (identities.has(id))
+ if (ids.has(id))
throw new CodexSecurityError(
`Duplicate source finding id ${JSON.stringify(id)}.`,
);
- identities.add(id);
- finding.evidence = validateExternalEvidence(finding.evidence);
+ ids.add(id);
+ finding.evidence = validateEvidence(finding.evidence);
+ }
+}
+
+function historicalPublication(
+ native: ExternalFindings | undefined,
+): PublicationSelection["original"] {
+ if (!native) return;
+ try {
+ // Older SDKs detached all raw records before Cloud validation added defaults.
+ for (const finding of native.findings)
+ if (finding.evidence.source_data !== undefined)
+ finding.evidence.source_data = structuredClone(
+ finding.evidence.source_data,
+ );
+ validatePublicationFindings(native, validatePreviousCloudEvidence);
+ const identity = publicationIdentity(native);
+ let hasBoxedScalars = false;
+ const wire = canonicalJson(identity, undefined, (value) => {
+ if (!isBoxedPrimitive(value)) return false;
+ hasBoxedScalars = true;
+ return true;
+ });
+ return { parsed: native, identity, wire, hasBoxedScalars };
+ } catch {
+ // The caller may export valid JSON even if the native candidate cannot be
+ // validated or hashed, for example when an unused row property holds a BigInt.
+ return;
+ }
+}
+
+function normalizedPublicationInput(
+ input: ExternalFindings,
+): PublicationSelection {
+ const nativeDiagnostics = new Map<number, Error>();
+ let native: ExternalFindings | undefined;
+ let plainJson = true;
+ try {
+ native = previousNormalizedFindings(input, nativeDiagnostics, (value) => {
+ if (!plainJson) return;
+ const prototype = Object.getPrototypeOf(value);
+ plainJson =
+ (prototype === null ||
+ prototype === Object.prototype ||
+ prototype === Array.prototype) &&
+ !("toJSON" in value);
+ });
+ } catch {
+ /* A valid JSON export can contain an uncloneable value. */
+ }
+ let priorIds = native ? undefined : previousFindingIds(input);
+ let current: ExternalFindings;
+ let freshError: Error | undefined;
+ try {
+ current = snapshotNormalizedFindings(input, undefined, "cloud");
+ } catch (error) {
+ if (
+ (!isNativeError(error) && !(error instanceof Error)) ||
+ !native?.findings.length
+ )
+ throw error;
+ freshError = error;
+ current = { read: native.read, findings: [], excluded: [] };
}
- const identity =
- current.report === undefined
- ? current.findings
- : { findings: current.findings, report: reportForJson(current.report) };
+ if (native && nativeDiagnostics.size) {
+ const historical = native.report!["errors"] as Error[];
+ const exported = current.report?.["errors"];
+ for (const [index, source] of nativeDiagnostics) {
+ const message = historical[index]!.message;
+ let diagnostic: unknown = Array.isArray(exported) && exported[index];
+ // A failed export may never reach the report. Its hooks have already run,
+ // so check the original descriptors again before native cloning reads a stack.
+ if (freshError && previousDiagnosticMessage(source) === message)
+ diagnostic = source;
+ if (
+ isNativeError(diagnostic) &&
+ Object.getPrototypeOf(diagnostic) === Object.getPrototypeOf(source)
+ ) {
+ try {
+ const copy = structuredClone(diagnostic);
+ if (copy.message === message) {
+ historical[index] = copy;
+ continue;
+ }
+ } catch {
+ // A failed native clone cannot establish a historical report.
+ }
+ }
+ if (freshError) throw freshError;
+ priorIds = previousFindingIds(native);
+ native = undefined;
+ break;
+ }
+ }
+ // A Cloud depth error can interrupt a local reader's JSON export. Its plain,
+ // independently captured output can still verify the original reader selection.
+ const readerOutput = freshError
+ ? freshError instanceof CloudEvidenceDepthError &&
+ plainJson &&
+ !nativeDiagnostics.size &&
+ native
+ : current;
+ const reader = readerOutput
+ ? previousReaderSelection(input, readerOutput, native ?? null)
+ : undefined;
+ if (reader) native = reader.snapshot;
+ const original = historicalPublication(native);
+ try {
+ validatePublicationFindings(current, validateExternalEvidence);
+ } catch (error) {
+ if (!(error instanceof Error) || !original?.parsed.findings.length)
+ throw error;
+ freshError = error;
+ }
+ if (freshError) {
+ if (!original) throw freshError;
+ // Only the old selection can be reviewed when fresh validation failed.
+ current = { read: original.parsed.read, findings: [], excluded: [] };
+ }
+ const identity = publicationIdentity(current);
return {
current,
legacy: current,
currentIdentity: identity,
legacyIdentity: identity,
+ original,
+ freshError,
+ unprovenIdPrefixes: reader?.unprovenIdPrefixes,
+ uncertainFindingIds: reader?.uncertainFindingIds,
+ unprovenOccurrenceCount: reader?.unprovenOccurrenceCount,
+ knownFindings: reader?.knownFindings,
+ potentialIds:
+ reader?.sourceFindingIds ??
+ (original
+ ? undefined
+ : (native?.findings.map((row) => row.source_finding_id) ?? priorIds)),
};
}
+// The old hash represented Number objects as {}. Unpublished row properties
+// and report metadata cannot be recovered from the immutable saved request.
+function verifiableLegacyMetadata(
+ { report, findings }: ExternalFindings,
+ wireIdentity: boolean,
+): boolean {
+ let verifiable = true;
+ // Only values in these records can be old scalar projections. Their record
+ // containers are required by the normalized selection, including an empty report.
+ const { errors, ...metadata } = report ?? {};
+ JSON.stringify(
+ [
+ Object.values(metadata),
+ // The original hash used the serialized form of report.errors arrays.
+ Array.isArray(errors) ? undefined : errors,
+ findings.map(({ evidence: _, ...fields }) => Object.values(fields)),
+ ],
+ (_key, value: unknown) => {
+ if (isRecord(value) && !(wireIdentity && isBoxedPrimitive(value))) {
+ // {} and string-index records can both be the old form of a scalar.
+ let index = 0;
+ if (
+ Object.keys(value).every((key) => {
+ const item = value[key];
+ return (
+ item === undefined ||
+ (key === String(index++) &&
+ typeof item === "string" &&
+ item.length === 1)
+ );
+ })
+ )
+ verifiable = false;
+ }
+ return value;
+ },
+ );
+ return verifiable;
+}
+
async function preparePublicationInput(
- input: ExternalPublicationInput,
+ input: PublicationSelection,
options: NormalizedExternalPublicationOptions,
dependencies: ExternalPublicationDependencies,
): Promise<PreparedExternalPublication> {
@@ -459,18 +830,28 @@ async function preparePublicationInput(
);
}
progress({ phase: "reading", completed: 0 });
- const {
+ let {
current: parsed,
legacy: legacyParsed,
currentIdentity,
legacyIdentity,
} = input;
- if (parsed.findings.length === 0) {
+ function noFindings(): never {
const diagnostics = reportForJson(parsed.report)?.["errors"];
throw new CodexSecurityError(
`No supported findings are ready to publish. ${parsed.excluded.map((item) => `Item ${item.position}: ${item.reason}`).join(" ")}${Array.isArray(diagnostics) && diagnostics.length ? ` Scanner diagnostics: ${JSON.stringify(diagnostics)}` : ""}`,
);
}
+ if (
+ !parsed.findings.length &&
+ !input.original?.parsed.findings.length &&
+ input.potentialIds !== null &&
+ !input.potentialIds?.length &&
+ !input.uncertainFindingIds?.length &&
+ !input.unprovenOccurrenceCount &&
+ !input.unprovenIdPrefixes?.length
+ )
+ noFindings();
progress({ phase: "discovering", completed: 0 });
const environment = dependencies.environment ?? process.env;
requireExternalImports(environment);
@@ -564,187 +945,755 @@ async function preparePublicationInput(
destination.repo_connector_id,
source,
];
- const inputDigest = hash("sha256", canonicalJson(currentIdentity));
- const legacyInputDigest = hash("sha256", canonicalJson(legacyIdentity));
- function validateSaved(serialized: string, legacy = false): SavedSubmission {
- const content = JSON.parse(serialized) as SavedSubmission;
+ function scopeKey(identity: unknown, serialize = wireJson) {
+ return jsonDigest([...checkpointIdentity, identity], serialize);
+ }
+ function localeKey(rows: ExternalFindings["findings"]) {
+ return hash(
+ "sha256",
+ canonicalJson([...checkpointIdentity, rows], (left, right) =>
+ left.localeCompare(right),
+ ),
+ );
+ }
+ const original = input.original;
+ const originalParsed = original?.parsed ?? parsed;
+ const originalIdentity = original?.identity ?? currentIdentity;
+ const hasBoxedScalars = original?.hasBoxedScalars === true;
+ const changedCapture =
+ original && original.wire !== wireJson(currentIdentity);
+ const originalKey =
+ changedCapture || hasBoxedScalars
+ ? scopeKey(originalIdentity, canonicalJson)
+ : undefined;
+ let historicalOnly = false;
+ const originalWireKey = changedCapture
+ ? scopeKey(originalIdentity)
+ : undefined;
+ if (changedCapture && originalKey && originalWireKey) {
+ // A released SDK could stage either spelling. A modern approval for the
+ // fresh JSON can occupy the very same historical filename.
+ for (const pathKey of new Set([
+ originalKey,
+ originalWireKey,
+ localeKey(original.parsed.findings),
+ ])) {
+ const bytes = await readOptionalFile(
+ join(state, `${pathKey}.pending.json`),
+ );
+ if (bytes === undefined) continue;
+ const record = JSON.parse(bytes) as SavedSubmission | null;
+ if (
+ (record?.legacyApproval &&
+ (record.wireKey ?? pathKey) === originalWireKey) ||
+ (record?.wireKey === undefined &&
+ record?.inputProjection?.identityFormat !== 2 &&
+ (pathKey === originalKey ||
+ pathKey === originalWireKey ||
+ original.parsed.report === undefined ||
+ record?.inputProjection?.inputDigest ===
+ jsonDigest(originalIdentity, canonicalJson)))
+ ) {
+ parsed = legacyParsed = original.parsed;
+ currentIdentity = legacyIdentity = original.identity;
+ historicalOnly = true;
+ break;
+ }
+ }
+ }
+ if (input.freshError && !historicalOnly) throw input.freshError;
+ const inputDigest = jsonDigest(currentIdentity);
+ const legacyInputDigest =
+ currentIdentity === legacyIdentity
+ ? inputDigest
+ : jsonDigest(legacyIdentity);
+ function matchesSavedDestination(requests: FindingImportRequest[]): boolean {
+ return (
+ requests.length > 0 &&
+ requests.every(
+ (batch) =>
+ batch.repository.id === destination.id &&
+ batch.repository.repo_connector_id ===
+ destination.repo_connector_id &&
+ wireJson(batch.source) === wireJson(source),
+ )
+ );
+ }
+ async function retireIfReset(
+ content: SavedSubmission,
+ retire: () => Promise<void>,
+ ) {
+ if (
+ content.requests.some(
+ (batch) => batch.repository.reset_marker !== destination.reset_marker,
+ )
+ ) {
+ await retire();
+ throw new CloudImportError(
+ 409,
+ "The repository was reset after this submission. The saved request was retired without uploading. Review the destination and run the command again to approve a fresh publication.",
+ );
+ }
+ }
+ function readSaved(
+ content: SavedSubmission | null | undefined,
+ ): SavedSubmission {
if (
+ !content ||
content.accountId !== credentials.account_id ||
!Array.isArray(content.requests) ||
content.requests.length === 0
)
throw new Error("Saved import account or requests do not match.");
content.requests = content.requests.map(validateImportRequest);
- const savedFindings = submissionFindings(content.requests);
- const savedEvidence = canonicalJson(savedFindings);
+ if (!matchesSavedDestination(content.requests))
+ throw new Error("Saved import destination does not match.");
+ return content;
+ }
+
+ function validateSaved(
+ record: SavedSubmission | null | undefined,
+ legacy = false,
+ ): SavedSubmission {
+ const content = readSaved(record);
+ const saved = submissionFindings(content.requests);
+ const wire = wireJson(saved);
const expected = legacy ? legacyParsed.findings : parsed.findings;
- const matches =
- content.inputProjection !== undefined
- ? sameFindingSelection(savedFindings, expected) &&
- content.inputProjection?.inputDigest ===
- (legacy ? legacyInputDigest : inputDigest) &&
- content.inputProjection?.savedEvidenceDigest ===
- hash("sha256", savedEvidence)
- : savedEvidence === canonicalJson(expected);
- if (!matches)
+ const projection = content.inputProjection;
+ if (
+ projection &&
+ (!sameFindingSelection(saved, expected) ||
+ projection.inputDigest !== (legacy ? legacyInputDigest : inputDigest) ||
+ projection.savedEvidenceDigest !== hash("sha256", wire))
+ )
throw new Error(
"Saved import evidence does not match the selected input.",
);
- for (const submission of content.requests) {
- if (
- submission.repository.id !== destination.id ||
- submission.repository.repo_connector_id !==
- destination.repo_connector_id ||
- canonicalJson(submission.source) !== canonicalJson(source)
- )
- throw new Error("Saved import destination does not match.");
+ // Older file readers can have a different, explicitly bound projection.
+ // A corrected numeric checkpoint must carry the selected evidence verbatim.
+ const exactWire =
+ historicalOnly || hasBoxedScalars || content.wireKey !== undefined;
+ if (
+ (!projection || exactWire) &&
+ wire !== wireJson(exactWire ? expected.map(wireFinding) : expected)
+ )
+ throw new Error(
+ "Saved import evidence does not match the selected input.",
+ );
+ if (
+ !legacy &&
+ hasBoxedScalars &&
+ projection?.identityFormat !== 2 &&
+ !verifiableLegacyMetadata(originalParsed, true)
+ )
+ throw unmatchedPreviousNumbers();
+ return content;
+ }
+
+ // The original filename and digests identify the previous selection, but
+ // contained {} for every boxed number. Only its actual saved request can
+ // prove which numbers were approved.
+ function readPreviousNumbers(
+ record: SavedSubmission | null,
+ ): SavedSubmission | "different-wire" | undefined {
+ const content = readSaved(record);
+ const projection = content.inputProjection;
+ const savedWire = wireJson(submissionFindings(content.requests));
+ const wireDigest = hash("sha256", savedWire);
+ if (projection?.inputDigest !== jsonDigest(originalIdentity, canonicalJson))
+ throw unmatchedPreviousNumbers();
+ if (projection.identityFormat === 2) {
+ // A current writer may own the old filename for an ordinary object.
+ if (projection.savedEvidenceDigest === wireDigest) return undefined;
+ throw unmatchedPreviousNumbers();
+ }
+ const selected = originalParsed.findings.map(wireFinding);
+ if (projection.savedEvidenceDigest !== jsonDigest(selected, canonicalJson))
+ throw unmatchedPreviousNumbers();
+ if (savedWire !== wireJson(selected)) {
+ // A genuinely empty object at this key is a separate ordinary selection.
+ if (wireDigest === projection.savedEvidenceDigest) return undefined;
+ return "different-wire";
}
+ if (
+ (changedCapture && !historicalOnly) ||
+ !verifiableLegacyMetadata(originalParsed, false)
+ )
+ throw unmatchedPreviousNumbers();
+ content.legacyApproval = true;
+ content.inputProjection = {
+ identityFormat: 2,
+ inputDigest,
+ savedEvidenceDigest: wireDigest,
+ };
return content;
}
- const key = hash(
- "sha256",
- canonicalJson([...checkpointIdentity, currentIdentity]),
- );
- const pendingPath = join(state, `${key}.pending.json`);
- const historicalKey = hash(
- "sha256",
- canonicalJson(
- [...checkpointIdentity, legacyParsed.findings],
- (left, right) => left.localeCompare(right),
- ),
- );
- const legacyKeys = new Set([
- hash("sha256", canonicalJson([...checkpointIdentity, legacyIdentity])),
- historicalKey,
- ]);
- async function rejectUnidentifiedLegacyCheckpoint(): Promise<void> {
- const entries = await readdir(state, { withFileTypes: true }).catch(
- (error: NodeJS.ErrnoException) => {
- if (error.code === "ENOENT") return [];
- throw error;
- },
+ function unmatchedPreviousNumbers() {
+ return new CodexSecurityError(
+ "The older saved import cannot be matched to the selected input. Its checkpoint was preserved without uploading.",
);
- if (entries.length) await validatePreparedOutputDir(state, undefined, true);
- for (const entry of entries) {
- if (!entry.isFile() || !/^[a-f0-9]{64}\.pending\.json$/u.test(entry.name))
- continue;
- const candidateKey = entry.name.slice(0, -".pending.json".length);
- if (candidateKey === key || legacyKeys.has(candidateKey)) continue;
- const serialized = await readOptionalFile(join(state, entry.name));
- if (serialized === undefined) continue;
- try {
- // Locale-dependent filenames predate input projections. Later formats
- // must remain isolated by their deployment-bound checkpoint keys.
- const content = JSON.parse(serialized) as SavedSubmission | null;
- if (!content || content.inputProjection !== undefined) continue;
- validateSaved(serialized, true);
- } catch {
- continue;
- }
- // These older bodies contain no deployment identity. Do not guess a
- // destination or replace their immutable request with a fresh publication.
- throw new CodexSecurityError(
- "An older pending import matches this input, but its checkpoint identity could not be established. Resume it using the original deployment and locale before publishing this selection. The saved request was preserved without uploading.",
- );
+ }
+ const key = scopeKey(currentIdentity);
+ const stagingKey = originalKey ?? key;
+ const canonicalPath = join(state, `${key}.pending.json`);
+ const lookupPath = join(state, `${key}.lookup.json`);
+ // A lookup only suggests a regular file to validate. A missing or stale hint
+ // leaves the same on-disk discovery available as before.
+ async function readOptionalRegularFile(path: string) {
+ try {
+ if ((await lstat(path)).isFile()) return await readFile(path, "utf8");
+ } catch {
+ // Optional discovery must not veto an independently verified checkpoint.
}
}
+ async function lookupFileKey() {
+ const serialized = await readOptionalRegularFile(lookupPath);
+ return serialized?.trim().match(/^"([a-f0-9]{64})"$/u)?.[1];
+ }
+
+ const historicalKey = localeKey(legacyParsed.findings);
+ const legacyIdentityKey = scopeKey(legacyIdentity);
+ const legacyKeys = new Set([legacyIdentityKey, historicalKey]);
+ function underLocks<T>(keys: string[], operation: () => Promise<T>) {
+ return withImportLocks(state, keys, dependencies.signal, operation);
+ }
let recovered: SavedSubmission | undefined;
for (const legacyKey of legacyKeys) {
- if (legacyKey === key) continue;
+ if (legacyKey === key || legacyKey === originalKey) continue;
const legacyPath = join(state, `${legacyKey}.pending.json`);
const captured = await readOptionalFile(legacyPath);
if (captured !== undefined) {
// Preserve requests staged by the earlier locale-based filename format.
// Hold both publication locks while moving the original checkpoint.
- await withImportLock(state, key, dependencies.signal, async () =>
- withImportLock(state, legacyKey, dependencies.signal, async () => {
- const current = await readOptionalFile(pendingPath);
- const serialized = await readOptionalFile(legacyPath);
- if (current !== undefined) {
- recovered = validateSaved(current);
- if (serialized !== undefined) {
- // A crash may leave both copies after installing the canonical file.
- // Retire only an identical immutable submission, never another request.
- let duplicate: SavedSubmission | null = null;
- try {
- duplicate = JSON.parse(serialized) as SavedSubmission | null;
- } catch (error) {
- if (!(error instanceof SyntaxError)) throw error;
- }
- if (
- duplicate !== null &&
- typeof duplicate === "object" &&
- sameSubmission(duplicate, recovered)
- )
- await rm(legacyPath);
- }
- return;
+ await underLocks([key, legacyKey], async () => {
+ const current = await readOptionalFile(canonicalPath);
+ const serialized = await readOptionalFile(legacyPath);
+ let prior: SavedSubmission | undefined | null;
+ if (serialized !== undefined) {
+ try {
+ prior = JSON.parse(serialized) as SavedSubmission | null;
+ } catch (error) {
+ if (current === undefined || !(error instanceof SyntaxError))
+ throw error;
}
- // A competing resumer may have completed and removed both paths while
- // this caller waited. Retain its observed immutable request and receipts.
- const content = validateSaved(serialized ?? captured, true);
- if (!sameFindingSelection(legacyParsed.findings, parsed.findings))
- throw new CodexSecurityError(
- "The saved import selects different findings from this input. Resume the original selection before publishing this selection.",
- );
+ }
+ if (prior?.wireKey !== undefined) return;
+ if (current !== undefined) {
+ recovered = validateSaved(JSON.parse(current));
+ if (prior && sameSubmission(prior, recovered)) await rm(legacyPath);
+ return;
+ }
+ // A report-free approval at the locale-era filename can be a separate input.
+ if (
+ legacyKey !== legacyIdentityKey &&
+ prior?.inputProjection &&
+ prior.inputProjection.inputDigest !== legacyInputDigest
+ ) {
+ const projection = prior.inputProjection;
+ const rows = legacyParsed.findings;
if (
- content.requests.some(
- (submission) =>
- submission.repository.reset_marker !== destination.reset_marker,
+ projection.identityFormat === 2 ||
+ (projection.inputDigest === jsonDigest(rows) &&
+ projection.savedEvidenceDigest ===
+ jsonDigest(submissionFindings(prior.requests)))
+ )
+ return;
+ // The released SDK hashed report-free rows as an array and boxed scores as {}.
+ const originalRows = original?.parsed.findings ?? rows;
+ if (
+ hasBoxedScalars &&
+ legacyParsed.report !== undefined &&
+ legacyKey === scopeKey(originalRows, canonicalJson) &&
+ projection.inputDigest ===
+ jsonDigest(originalRows, canonicalJson) &&
+ projection.savedEvidenceDigest ===
+ jsonDigest(originalRows.map(wireFinding), canonicalJson) &&
+ sameFindingSelection(
+ submissionFindings(readSaved(prior).requests),
+ rows,
)
+ )
+ return;
+ }
+ const content = validateSaved(
+ serialized === undefined ? JSON.parse(captured) : prior,
+ true,
+ );
+ if (!sameFindingSelection(legacyParsed.findings, parsed.findings))
+ throw new CodexSecurityError(
+ "The saved import selects different findings from this input. Resume the original selection before publishing this selection.",
+ );
+ await retireIfReset(content, async () => {
+ if (serialized !== undefined) await rm(legacyPath);
+ });
+ const savedEvidenceDigest =
+ content.inputProjection?.savedEvidenceDigest ??
+ jsonDigest(legacyParsed.findings);
+ const rebindProjection =
+ content.inputProjection !== undefined ||
+ savedEvidenceDigest !== inputDigest;
+ if (rebindProjection) {
+ if (content.inputProjection?.identityFormat !== 2)
+ content.legacyApproval = true;
+ content.inputProjection = {
+ ...content.inputProjection,
+ identityFormat: 2,
+ inputDigest,
+ savedEvidenceDigest,
+ };
+ }
+ recovered = content;
+ if (serialized === undefined) return;
+ if (!rebindProjection) {
+ await rename(legacyPath, canonicalPath);
+ } else {
+ // Bind the new input projection to the original immutable evidence,
+ // so later inventory reordering still resumes these exact requests.
+ await writeAtomicJson(canonicalPath, content);
+ await rm(legacyPath);
+ }
+ });
+ }
+ }
+
+ interface Checkpoint {
+ fileKey: string;
+ content: SavedSubmission;
+ historical: boolean;
+ checkedCanonical?: true;
+ }
+ // Some getter-backed batches cannot reproduce a released filename. Their IDs
+ // can stop a conflicting upload, but never authorize replay of a saved request.
+ const potentialIds =
+ input.potentialIds === undefined && changedCapture
+ ? original.parsed.findings.map((row) => row.source_finding_id)
+ : input.potentialIds;
+ const unprovenPrefixes = new Set(input.unprovenIdPrefixes);
+ const uncertainIds = new Set(input.uncertainFindingIds);
+ const uncertainMembership =
+ input.unprovenOccurrenceCount ||
+ uncertainIds.size > 0 ||
+ unprovenPrefixes.size > 0;
+ function uncertainId(id: string) {
+ return (
+ uncertainIds.has(id) ||
+ unprovenPrefixes.has(id.replace(/\/[a-f0-9]{64}$/u, "/"))
+ );
+ }
+ const stableIds = uncertainMembership
+ ? (
+ potentialIds ?? parsed.findings.map((row) => row.source_finding_id)
+ ).filter((id) => id === undefined || !uncertainId(id))
+ : undefined;
+ const knownMembership = input.unprovenOccurrenceCount
+ ? new Set(stableIds)
+ : undefined;
+ const knownRows = new Map(
+ input.knownFindings?.map((row) => [
+ row.source_finding_id,
+ wireFinding(row),
+ ]),
+ );
+ let originalInputDigest: string | undefined;
+ const cannotCaptureOriginal =
+ !input.original &&
+ (potentialIds === null || !!(potentialIds?.length || uncertainMembership));
+ const possibleBufferExport =
+ options.provider === "github" && original && !changedCapture;
+ function potentialOriginal(record: SavedSubmission, fileKey: string) {
+ const projection = record.inputProjection;
+ if (record.accountId !== credentials.account_id) return false;
+ if (!projection) {
+ // Locale-era requests had no projection. Their scoped occurrence IDs
+ // can stop an opaque retry but cannot authorize replay.
+ if (!cannotCaptureOriginal || record.wireKey !== undefined) return false;
+ } else if (
+ !/^[a-f0-9]{64}$/u.test(projection.inputDigest) ||
+ !/^[a-f0-9]{64}$/u.test(projection.savedEvidenceDigest) ||
+ (projection.identityFormat === 2 && !record.legacyApproval)
+ )
+ return false;
+ try {
+ // Older evidence may fail today's limits. Scope and selection can still
+ // identify an outstanding approval and prevent a duplicate submission.
+ if (!matchesSavedDestination(record.requests)) return false;
+ const rows = submissionFindings(record.requests);
+ const savedWire = wireJson(rows);
+ const evidenceMatches =
+ projection &&
+ projection.savedEvidenceDigest === hash("sha256", savedWire);
+ if (evidenceMatches) {
+ // Matching old digests reconstruct its scoped filename. A report from
+ // this caller is useful only when the saved digest independently binds it.
+ const reportFree =
+ projection.inputDigest === projection.savedEvidenceDigest;
+ const identity = reportFree
+ ? rows
+ : parsed.report !== undefined
+ ? { findings: rows, report: reportForJson(parsed.report) }
+ : undefined;
+ if (
+ identity &&
+ (reportFree || projection.inputDigest === jsonDigest(identity))
+ ) {
+ if (
+ record.wireKey
+ ? record.wireKey !== scopeKey(identity)
+ : fileKey !== scopeKey(identity, canonicalJson) &&
+ fileKey !== localeKey(rows)
+ )
+ return false;
+ if (
+ possibleBufferExport &&
+ projection.inputDigest !== inputDigest &&
+ sameLegacyJson(identity, currentIdentity, previousBufferJson)
) {
- if (serialized !== undefined) await rm(legacyPath);
- throw new CloudImportError(
- 409,
- "The repository was reset after this submission. The saved request was retired without uploading. Review the destination and run the command again to approve a fresh publication.",
- );
- }
- const savedEvidenceDigest =
- content.inputProjection?.savedEvidenceDigest ??
- hash("sha256", canonicalJson(legacyParsed.findings));
- const rebindProjection =
- content.inputProjection !== undefined ||
- savedEvidenceDigest !== inputDigest;
- if (rebindProjection)
- content.inputProjection = { inputDigest, savedEvidenceDigest };
- recovered = content;
- if (serialized === undefined) return;
- if (!rebindProjection) {
- await rename(legacyPath, pendingPath);
- } else {
- // Bind the new input projection to the original immutable evidence,
- // so later inventory reordering still resumes these exact requests.
- await writeAtomicJson(pendingPath, content);
- await rm(legacyPath);
+ return true;
}
- }),
+ }
+ if (
+ !record.wireKey &&
+ reportFree &&
+ original &&
+ original.wire !== savedWire
+ )
+ return false;
+ }
+ if (record.wireKey && !cannotCaptureOriginal) return false;
+ if (
+ projection &&
+ original &&
+ fileKey !== stagingKey &&
+ fileKey !== originalWireKey &&
+ projection.inputDigest ===
+ (originalInputDigest ??= jsonDigest(
+ original.identity,
+ canonicalJson,
+ )) &&
+ fileKey !== localeKey(original.parsed.findings) &&
+ projection.savedEvidenceDigest ===
+ jsonDigest(
+ original.parsed.findings.map(wireFinding),
+ canonicalJson,
+ ) &&
+ savedWire === wireJson(original.parsed.findings.map(wireFinding))
+ )
+ return false;
+ if (stableIds) {
+ // An opaque occurrence may have had a different old ID. Compare all
+ // definite members and count the remaining IDs without claiming them.
+ const certain = rows.filter(
+ (row) => !uncertainId(row.source_finding_id),
+ );
+ const stableSaved = knownMembership
+ ? certain.filter((row) => knownMembership.has(row.source_finding_id))
+ : certain;
+ if (
+ certain.length - stableSaved.length >
+ (input.unprovenOccurrenceCount ?? 0)
+ )
+ return false;
+ return (
+ rows.length > 0 &&
+ stableSaved.length === stableIds.length &&
+ stableSaved.every((row, i) => {
+ const known = knownRows.get(row.source_finding_id);
+ return (
+ row.source_finding_id === stableIds[i] &&
+ (!known || wireJson(row) === wireJson(known))
+ );
+ })
+ );
+ }
+ const candidates = originalParsed.findings;
+ const ids =
+ potentialIds === undefined
+ ? parsed.findings.map((row) => row.source_finding_id)
+ : potentialIds;
+ if (
+ ids &&
+ (!ids.length ||
+ rows.length !== ids.length ||
+ !rows.every(
+ (row, i) =>
+ ids[i] === undefined || row.source_finding_id === ids[i],
+ ))
+ )
+ return false;
+ // An opaque serializer can add a report or replace every visible field.
+ // Only scoped, independently known occurrences can rule out its old request.
+ if (cannotCaptureOriginal || !projection) return true;
+ if (
+ projection.inputDigest === projection.savedEvidenceDigest &&
+ parsed.report !== undefined
+ )
+ return false;
+ if (potentialIds === undefined && (hasBoxedScalars || evidenceMatches))
+ return false;
+ return (
+ !sameFindingSelection(rows, candidates) ||
+ rows.every((row, i) => {
+ const { source_data: savedData, ...savedFields } = row.evidence;
+ const { source_data: selectedData, ...selectedFields } =
+ candidates[i]!.evidence;
+ if (wireJson(savedFields) !== wireJson(selectedFields)) return false;
+ // Once the original SDK discarded boxed positions, a plain JSON export
+ // alone cannot attribute a new numeric spelling to that saved request.
+ return sameLegacyJson(
+ savedData,
+ selectedData,
+ (value) => typeof value === "number",
+ );
+ })
);
+ } catch {
+ return false;
}
}
- const serialized = await readOptionalFile(pendingPath);
- const saved =
- serialized === undefined ? recovered : validateSaved(serialized);
- if (!saved) await rejectUnidentifiedLegacyCheckpoint();
- if (saved) {
- const content = saved;
- for (const submission of content.requests) {
- if (submission.repository.reset_marker !== destination.reset_marker) {
- // The old request is never sent after reset. A subsequent explicit
- // invocation prepares a fresh submission and asks for approval again.
- await withImportLock(state, key, dependencies.signal, async () => {
- const current = await readOptionalFile(pendingPath);
- if (
- current !== undefined &&
- sameSubmission(JSON.parse(current) as SavedSubmission, content)
- )
- await rm(pendingPath);
- });
- throw new CloudImportError(
- 409,
- "The repository was reset after this submission. The saved request was retired without uploading. Review the destination and run the command again to approve a fresh publication.",
+ async function selectedCheckpoint(knownFileKey?: string, preparing = false) {
+ const directory = await validatePreparedOutputDir(
+ state,
+ undefined,
+ true,
+ ).catch((error: NodeJS.ErrnoException) => {
+ if (error.code === "ENOENT") return undefined;
+ throw error;
+ });
+ if (!directory) return undefined;
+ const known = new Set(
+ [key, originalKey, knownFileKey].filter((value) => value !== undefined),
+ );
+ let unidentified = false;
+ let unprovenOriginal = false;
+ let incomplete = false;
+ let stagingPathOccupied = false;
+ async function readCandidate(
+ fileKey: string,
+ selected?: Checkpoint,
+ contents?: string,
+ ): Promise<Checkpoint | undefined> {
+ const path = join(state, `${fileKey}.pending.json`);
+ const serialized =
+ contents ??
+ (selected?.fileKey === key && !known.has(fileKey)
+ ? await readOptionalRegularFile(path)
+ : await readOptionalFile(path));
+ if (serialized === undefined) {
+ if (!known.has(fileKey)) incomplete = true;
+ return selected;
+ }
+ if (fileKey === stagingKey) stagingPathOccupied = true;
+ let record: SavedSubmission | null;
+ try {
+ record = JSON.parse(serialized) as SavedSubmission | null;
+ } catch (error) {
+ if (known.has(fileKey)) throw error;
+ incomplete = true;
+ return selected;
+ }
+ // A migrated checkpoint records the wire identity it belongs to even
+ // when a separate input hashes to its historical filename.
+ const tag = record?.wireKey;
+ const tagged = tag !== undefined;
+ const ownerKey = tagged ? tag : fileKey;
+ const historical = !tagged && fileKey === originalKey && fileKey !== key;
+ if (ownerKey !== key && !historical) {
+ if (
+ record &&
+ (!tagged || cannotCaptureOriginal || possibleBufferExport) &&
+ potentialOriginal(record, fileKey)
+ )
+ unprovenOriginal = true;
+ // Some early locale filenames carry no deployment identity. Never guess their target.
+ if (
+ !tagged &&
+ record &&
+ !record.inputProjection &&
+ !legacyKeys.has(fileKey)
+ ) {
+ try {
+ // A different locale cannot authorize replay, even when the
+ // original native capture proves these rows were selected.
+ if (
+ !original ||
+ wireJson(submissionFindings(readSaved(record).requests)) !==
+ wireJson(original.parsed.findings.map(wireFinding))
+ )
+ validateSaved(record, true);
+ unidentified = true;
+ } catch {
+ /* Unrelated input. */
+ }
+ }
+ return selected;
+ }
+ const content = historical
+ ? readPreviousNumbers(record)
+ : validateSaved(record);
+ if (content === "different-wire") {
+ // Another spelling may already have a verified checkpoint for this wire value.
+ unprovenOriginal = true;
+ return selected;
+ }
+ if (!content) return selected;
+ if (selected) {
+ const conflict = new CodexSecurityError(
+ "Two saved imports match this input. Their original requests and receipts were preserved without uploading.",
);
+ if (!preparing) throw conflict;
+ // The two unlocked reads may have straddled completion of the first request.
+ return await underLocks([key, selected.fileKey, fileKey], async () => {
+ const first = await readCandidate(selected.fileKey);
+ if (first) {
+ if (await readCandidate(fileKey)) throw conflict;
+ if (sameSubmission(first.content, selected.content)) return first;
+ }
+ return completedCheckpoint(selected);
+ });
+ }
+ return { fileKey, content, historical };
+ }
+ let selected: Checkpoint | undefined;
+ for (const fileKey of known)
+ selected = await readCandidate(fileKey, selected);
+ // A known file that disappeared can be checked against its completed receipt.
+ if (!selected && knownFileKey !== undefined) return undefined;
+ const hint = await lookupFileKey();
+ if (hint && !known.has(hint)) {
+ const contents = await readOptionalRegularFile(
+ join(state, `${hint}.pending.json`),
+ );
+ if (contents !== undefined) {
+ selected = await readCandidate(hint, selected, contents);
+ known.add(hint);
}
}
+ // A released publisher could stage the canonical request while another SDK
+ // tagged its old filename but had not yet written the optional lookup.
+ if (
+ selected &&
+ !cannotCaptureOriginal &&
+ !selected.historical &&
+ (selected.fileKey !== key ||
+ selected.content.canonicalChecked ||
+ selected.content.inputProjection?.identityFormat === 2)
+ )
+ return selected;
+ const entries = await readdir(state, { withFileTypes: true }).catch(
+ (error: NodeJS.ErrnoException) => {
+ if (error.code === "ENOENT") return [];
+ throw error;
+ },
+ );
+ for (const entry of entries) {
+ if (!entry.isFile() || !/^[a-f0-9]{64}\.pending\.json$/u.test(entry.name))
+ continue;
+ const fileKey = entry.name.slice(0, -".pending.json".length);
+ if (!known.has(fileKey))
+ selected = await readCandidate(fileKey, selected);
+ }
+ if (unprovenOriginal && (!selected || cannotCaptureOriginal))
+ throw unmatchedPreviousNumbers();
+ if (!selected && stagingPathOccupied) throw occupiedCheckpoint();
+ if (!selected && unidentified)
+ throw new CodexSecurityError(
+ "An older pending import matches this input, but its checkpoint identity could not be established. Resume it using the original deployment and locale before publishing this selection. The saved request was preserved without uploading.",
+ );
+ if (selected?.fileKey === key && !preparing && !incomplete)
+ selected.checkedCanonical = true;
+ return selected;
+ }
+ async function readCompletedPublication(
+ requests: FindingImportRequest[],
+ observedKey: string,
+ ) {
+ for (const resultKey of new Set([key, observedKey])) {
+ const serialized = await readOptionalFile(
+ join(state, `${resultKey}.result.json`),
+ );
+ if (serialized === undefined) continue;
+ const completed = JSON.parse(
+ serialized,
+ ) as ExternalPublicationResult | null;
+ if (
+ (completed?.status === "complete" || completed?.status === "partial") &&
+ Array.isArray(completed.receipts) &&
+ completed.receipts.length === requests.length &&
+ completed.receipts.every(
+ (receipt, i) => receipt?.id === requests[i]!.request_id,
+ )
+ )
+ return {
+ receipts: completed.receipts.map((receipt, i) =>
+ validatePublicationReceipt(receipt, requests[i]!),
+ ),
+ verified: completed.verified,
+ };
+ }
+ }
+ async function completedCheckpoint(observed: Checkpoint) {
+ if (
+ await readCompletedPublication(
+ observed.content.requests,
+ observed.fileKey,
+ )
+ )
+ return observed;
+ throw new CodexSecurityError(
+ "The saved publication changed while preparing this input. Review it again.",
+ );
+ }
+ function occupiedCheckpoint() {
+ return new CodexSecurityError(
+ "Finish the other pending import at this checkpoint first, then review this input again.",
+ );
+ }
+ let selected = await selectedCheckpoint(recovered ? key : undefined, true);
+ const fileKey = selected?.fileKey ?? (recovered ? key : stagingKey);
+ const pendingPath = join(state, `${fileKey}.pending.json`);
+ async function rememberFile() {
+ if (fileKey !== key && (await lookupFileKey()) !== fileKey)
+ await writeAtomicJson(lookupPath, fileKey).catch(() => undefined);
+ }
+ async function removePending() {
+ await rm(pendingPath, { force: true });
+ if (fileKey !== key && (await lookupFileKey()) === fileKey)
+ await rm(lookupPath, { force: true }).catch(() => undefined);
+ }
+
+ const lockKeys = [key, fileKey, stagingKey];
+ if (selected?.historical) {
+ const observed = selected;
+ selected = await underLocks(lockKeys, async () => {
+ const current = await selectedCheckpoint(fileKey);
+ if (
+ !current ||
+ current.fileKey !== fileKey ||
+ !sameSubmission(current.content, observed.content)
+ )
+ return completedCheckpoint(observed);
+ if (current.historical) {
+ current.content.wireKey = key;
+ await writeAtomicJson(pendingPath, current.content);
+ await rememberFile();
+ }
+ return current;
+ });
+ }
+ const saved = selected?.content ?? recovered;
+ if (historicalOnly && !saved) throw unmatchedPreviousNumbers();
+ if (!parsed.findings.length) noFindings();
+ if (saved) {
+ await retireIfReset(saved, () =>
+ underLocks(lockKeys, async () => {
+ const current = await readOptionalFile(pendingPath);
+ if (
+ current !== undefined &&
+ sameSubmission(JSON.parse(current) as SavedSubmission, saved)
+ )
+ await removePending();
+ }),
+ );
}
if (saved?.inspectorProjectMapRequired && !inspectorProjectMap)
throw new CodexSecurityError(
@@ -944,12 +1893,12 @@ async function preparePublicationInput(
accountId: credentials.account_id,
requests,
receipts: saved?.receipts,
+ legacyApproval: saved?.legacyApproval,
+ wireKey: saved?.wireKey ?? (fileKey === key ? undefined : key),
inputProjection: saved?.inputProjection ?? {
+ identityFormat: saved ? undefined : 2,
inputDigest,
- savedEvidenceDigest: hash(
- "sha256",
- canonicalJson(submissionFindings(requests)),
- ),
+ savedEvidenceDigest: jsonDigest(submissionFindings(requests)),
},
};
if (inspectorProjectMap) submission.inspectorProjectMapRequired = true;
@@ -965,7 +1914,7 @@ async function preparePublicationInput(
requests,
};
return {
- preview: structuredClone(preview),
+ preview: cloneImportPreview(preview),
async publish() {
progress({ phase: "waiting", completed: 0 });
const result = (
@@ -998,7 +1947,7 @@ async function preparePublicationInput(
read: parsed.read,
...(parsed.report === undefined
? {}
- : { report: structuredClone(parsed.report) }),
+ : snapshotNormalizedFindings({ report: parsed.report })),
ready: parsed.findings.length,
excluded: parsed.excluded,
receipts: acknowledged,
@@ -1010,25 +1959,67 @@ async function preparePublicationInput(
verified,
};
};
+ function interruptedPreview(
+ message: string,
+ cause?: unknown,
+ savedSubmission?: string,
+ ) {
+ // The expected file is gone or another publisher owns the lock. Use only
+ // the acknowledgements this caller saw when it prepared the request.
+ const known = submission.receipts ?? [];
+ if (known.length > submission.requests.length)
+ throw new CodexSecurityError(
+ "Saved publication has more receipts than requests.",
+ );
+ const acknowledged = known.map((receipt, index) =>
+ validatePublicationReceipt(receipt, submission.requests[index]!),
+ );
+ return new ExternalPublicationError(
+ message,
+ {
+ ...result(acknowledged, 0),
+ status: "interrupted",
+ error: message,
+ ...(savedSubmission === undefined ? {} : { savedSubmission }),
+ },
+ cause,
+ );
+ }
let acquired = false;
- return await withImportLock(state, key, dependencies.signal, async () => {
+ return await underLocks(lockKeys, async () => {
acquired = true;
// Re-read under the lock to keep checkpoints from another publisher.
- const pending = await readOptionalFile(pendingPath);
- if (pending === undefined && !saved)
- await rejectUnidentifiedLegacyCheckpoint();
- const current =
- pending === undefined
- ? submission
- : (JSON.parse(pending) as SavedSubmission);
+ const selected = await selectedCheckpoint(
+ saved ? fileKey : undefined,
+ ).catch((error: unknown) => {
+ if (!submission.receipts?.length) throw error;
+ throw interruptedPreview(errorMessage(error), error);
+ });
+ const pending = selected?.content;
+ const current = pending ?? submission;
const adoptProjectMap =
submission.inspectorProjectMapRequired &&
current.inspectorProjectMapRequired === undefined;
if (adoptProjectMap) current.inspectorProjectMapRequired = true;
- if (!sameSubmission(current, submission))
- throw new CodexSecurityError(
- "Another publication prepared this input. Run the command again to review and resume that saved request.",
- );
+ if (
+ !selected ||
+ selected.fileKey !== fileKey ||
+ selected.historical ||
+ !sameSubmission(current, submission)
+ ) {
+ const message = selected
+ ? "Another publication prepared this input. Run the command again to review and resume that saved request."
+ : "The saved publication changed after preview. Review its result before publishing again.";
+ if (saved) {
+ const completed = await readCompletedPublication(requests, fileKey);
+ if (completed)
+ return result(completed.receipts, completed.verified);
+ throw interruptedPreview(message);
+ }
+ if (selected) throw new CodexSecurityError(message);
+ if ((await readOptionalFile(pendingPath)) !== undefined)
+ throw occupiedCheckpoint();
+ }
const receipts = (current.receipts ?? []).map(validateImportReceipt);
if (receipts.length > requests.length)
throw new CodexSecurityError(
@@ -1038,10 +2029,15 @@ async function preparePublicationInput(
let verified = 0;
let persisted = pending !== undefined;
try {
+ if (selected?.checkedCanonical) current.canonicalChecked = true;
if (pending === undefined || adoptProjectMap) {
await writeAtomicJson(pendingPath, current);
persisted = true;
+ } else if (selected?.checkedCanonical) {
+ // The existing approval and receipts are usable if caching the scan fails.
+ await writeAtomicJson(pendingPath, current).catch(() => undefined);
}
+ await rememberFile();
progress({
phase: "uploading",
completed: 0,
@@ -1067,6 +2063,7 @@ async function preparePublicationInput(
// Readback can still resume after this checkpoint without replaying writes.
await writeAtomicJson(pendingPath, {
...submission,
+ canonicalChecked: current.canonicalChecked,
receipts,
});
}
@@ -1103,13 +2100,12 @@ async function preparePublicationInput(
report.version < item.version! ||
(report.version === item.version &&
(report.observation_id !== item.observation_id ||
- canonicalJson(report.evidence) !==
- canonicalJson(expected.evidence))) ||
+ wireJson(report.evidence) !== wireJson(expected.evidence))) ||
report.source_finding_id !== expected.source_finding_id ||
report.repo_id !== destination.id ||
report.repo_connector_id !== destination.repo_connector_id ||
report.environment_id !== batch.repository.environment_id ||
- canonicalJson(report.source) !== canonicalJson(source)
+ wireJson(report.source) !== wireJson(source)
)
throw new CodexSecurityError(
"Cloud readback did not match the saved finding identity.",
@@ -1125,11 +2121,11 @@ async function preparePublicationInput(
...completed,
report: reportForJson(completed.report),
});
- await rm(pendingPath, { force: true });
+ await removePending();
return completed;
} catch (error) {
if (error instanceof CloudImportError && error.status === 409) {
- await rm(pendingPath, { force: true });
+ await removePending();
persisted = false;
}
const message = `${error instanceof Error ? error.message : String(error)} ${error instanceof CloudImportError && error.status === 409 ? "The old submission was retired. Rediscover and approve a fresh publication." : persisted ? `Repeat the same command to resume the saved request. Saved submission: ${pendingPath}` : "No resumable request was saved. Correct the error and review the input before retrying."}`;
@@ -1146,28 +2142,13 @@ async function preparePublicationInput(
}
}).catch((error: unknown) => {
if (acquired || !dependencies.signal?.aborted) throw error;
- // The active publisher owns the checkpoint. Report only acknowledgements
- // already observed during preparation, without rewriting its files.
- const known = submission.receipts ?? [];
- if (known.length > requests.length)
- throw new CodexSecurityError(
- "Saved publication has more receipts than requests.",
- );
- const acknowledged = known.map((receipt, index) =>
- validatePublicationReceipt(receipt, requests[index]!),
- );
const message = saved
? `Publication canceled while waiting. Repeat the same command to resume the saved request. Saved submission: ${pendingPath}`
: "Publication canceled while waiting. No resumable request was saved by this invocation.";
- throw new ExternalPublicationError(
+ throw interruptedPreview(
message,
- {
- ...result(acknowledged, 0),
- status: "interrupted",
- error: message,
- ...(saved ? { savedSubmission: pendingPath } : {}),
- },
error,
+ saved ? pendingPath : undefined,
);
});
},
diff --git a/sdk/typescript/src/external-findings.ts b/sdk/typescript/src/external-findings.ts
index 619f2ef51..3738a72f4 100644
--- a/sdk/typescript/src/external-findings.ts
+++ b/sdk/typescript/src/external-findings.ts
@@ -1,5 +1,13 @@
+import { isNativeError, isNumberObject } from "node:util/types";
import { CodexSecurityError } from "./errors.js";
import type { ExternalPublicationOptions } from "./external-findings-publish.js";
+import {
+ checkEmittedJsonNumber,
+ checkReportNumber,
+ requireCloudEvidenceDepth,
+ requireJsonNumber,
+ type ExternalImportTarget,
+} from "./external-import-contract.js";
import type { ExternalFindingEvidence } from "./external-import-models.js";
/** A provider occurrence and its evidence, independent of a Codex assessment. */
@@ -32,6 +40,131 @@ export interface ExternalFindings {
}[];
}
+const isScannerError = Error.isError ?? isNativeError;
+export const nativeErrorStackGetter = Object.getOwnPropertyDescriptor(
+ new Error(),
+ "stack",
+)?.get;
+
+function cloneScannerDiagnostic(source: Error): Error {
+ const message = Object.getOwnPropertyDescriptor(source, "message");
+ const stack = Object.getOwnPropertyDescriptor(source, "stack");
+ try {
+ if (
+ (!message || Object.hasOwn(message, "value")) &&
+ (!stack?.get || stack.get === nativeErrorStackGetter)
+ )
+ return structuredClone(source);
+ // Keep the native error type and data-valued cause without running a
+ // caller-supplied message or stack getter while restoring diagnostics.
+ const cause = Object.getOwnPropertyDescriptor(source, "cause");
+ const clean = new Error(message?.value, { cause: cause?.value });
+ const name = Object.getOwnPropertyDescriptor(
+ Object.getPrototypeOf(source),
+ "name",
+ )?.value;
+ if (typeof name === "string") clean.name = name;
+ if (!message?.get && nativeErrorStackGetter) {
+ const nativeStack: unknown = nativeErrorStackGetter.call(source);
+ if (typeof nativeStack === "string") clean.stack = nativeStack;
+ }
+ return structuredClone(clean);
+ } catch {
+ // A noncloneable cause does not prevent JSON export.
+ return new Error(message?.value);
+ }
+}
+
+/** @internal Capture fresh input once, so preview, limits and storage use the same JSON. */
+export function snapshotNormalizedFindings<T extends Partial<ExternalFindings>>(
+ input: T,
+ checkEvidenceNumber = (value: number) => requireJsonNumber(value, "Evidence"),
+ target: ExternalImportTarget = "local",
+): T {
+ const fields = new Map<string, unknown>();
+ const errors: { path: string[]; error: Error }[] = [];
+ let root: unknown;
+
+ function captureField(field: string, observed: unknown): string | undefined {
+ const parents: { key: string; value: object }[] = [];
+ // The outer traversal already called this field's toJSON with its real key.
+ // Returning that value from the wrapper keeps JSON from calling it twice.
+ return JSON.stringify(
+ { toJSON: () => observed },
+ function (this: object, key, value: unknown) {
+ while (parents.length && parents.at(-1)!.value !== this) parents.pop();
+ const evidence = field === "findings" && parents[2]?.key === "evidence";
+ if (evidence && target === "cloud")
+ requireCloudEvidenceDepth(parents.length - 2, this, value);
+ if (isNumberObject(value)) value = +value;
+ if (field === "report") checkReportNumber(value);
+ if (evidence) checkEmittedJsonNumber(value, checkEvidenceNumber);
+ if (value !== null && typeof value === "object") {
+ if (value === root) {
+ const path = [field, ...parents.slice(1).map((p) => p.key)];
+ if (parents.length) path.push(key);
+ throw new TypeError(
+ `Converting circular structure to JSON at ${JSON.stringify(path)}`,
+ );
+ }
+ if (
+ field === "report" &&
+ parents[1]?.key === "errors" &&
+ Array.isArray(parents[1].value) &&
+ isScannerError(value)
+ ) {
+ errors.push({
+ path: [field, ...parents.slice(1).map((p) => p.key), key],
+ error: value,
+ });
+ }
+ parents.push({ key, value });
+ }
+ return value;
+ },
+ );
+ }
+
+ // Capture each member in native property order. A report and a findings
+ // payload can each fit in a string while their combined JSON does not.
+ const envelope = JSON.stringify(
+ input,
+ function (this: object, field, value: unknown) {
+ if (this !== root) {
+ root = value;
+ return value;
+ }
+ const json = captureField(field, value);
+ if (json === undefined) return undefined;
+ fields.set(field, JSON.parse(json));
+ return null;
+ },
+ );
+ const result = JSON.parse(envelope);
+ for (const [key, value] of fields) result[key] = value;
+ for (const { path, error: source } of errors) {
+ const key = path.pop()!;
+ let parent = result as unknown as Record<string, unknown>;
+ for (const field of path) parent = parent[field] as Record<string, unknown>;
+ const error = cloneScannerDiagnostic(source);
+ Object.defineProperties(
+ error,
+ Object.getOwnPropertyDescriptors(parent[key] as object),
+ );
+ parent[key] = error;
+ }
+ return result;
+}
+
+/** @internal Clone owned values without serializing the evidence again in every preview view. */
+export function cloneImportPreview<T extends ExternalFindings>(input: T): T {
+ if (input.report === undefined) return structuredClone(input);
+ return {
+ ...structuredClone({ ...input, report: undefined }),
+ ...snapshotNormalizedFindings({ report: input.report }),
+ };
+}
+
/** @internal Keep earlier provider projections only for immutable request replay. */
export interface ExternalPublicationInput {
current: ExternalFindings;
diff --git a/sdk/typescript/src/external-import-contract.ts b/sdk/typescript/src/external-import-contract.ts
index c7a1cb7e4..df39c7818 100644
--- a/sdk/typescript/src/external-import-contract.ts
+++ b/sdk/typescript/src/external-import-contract.ts
@@ -1,5 +1,5 @@
import { isIP } from "node:net";
-import { isNumberObject } from "node:util/types";
+import { isNumberObject, isProxy } from "node:util/types";
import Ajv2020, { type ValidateFunction } from "ajv/dist/2020.js";
import schema from "../schemas/external-findings.schema.json" with { type: "json" };
import type {
@@ -227,28 +227,57 @@ export const validateSourceReports =
validator<SourceReportPage>("SourceReportPage");
export const validateSourceReport = validator<SourceReport>("SourceReport");
-function requireJsonNumber(value: number, label: string): void {
- if (!Number.isFinite(value))
- throw new Error(`${label} must contain finite JSON numbers.`);
+// Normalize equivalent JSON numbers exactly. Keep very large exponents compact.
+export function canonicalNumber(token: string): string {
+ const [, sign = "", integer, fraction = "", exponent = "0"] =
+ /^(-?)(\d+)(?:\.(\d+))?(?:e([+-]?\d+))?$/iu.exec(token)!;
+ const significant = (integer + fraction).replace(/^0+/u, "");
+ if (!significant) return "0";
+ let end = significant.length;
+ while (significant[end - 1] === "0") end--;
+ const digits = significant.slice(0, end);
+ const point = BigInt(exponent) + BigInt(significant.length - fraction.length);
+ // Ordinary evidence follows JSON.stringify’s fixed/scientific notation thresholds.
+ if (point > -6n && point <= 21n) {
+ const split = Number(point);
+ if (split <= 0) return sign + "0." + "0".repeat(-split) + digits;
+ if (split >= digits.length) return sign + digits.padEnd(split, "0");
+ return sign + digits.slice(0, split) + "." + digits.slice(split);
+ }
+ const trailing = digits.slice(1);
+ const scientific = point - 1n;
+ return (
+ sign +
+ digits.charAt(0) +
+ (trailing ? "." + trailing : "") +
+ "e" +
+ (scientific >= 0n ? "+" : "") +
+ scientific
+ );
+}
+
+export function requireSafeJsonInteger(value: number, label: string): void {
if (Number.isInteger(value) && !Number.isSafeInteger(value))
throw new Error(
`${label} integers must be within JavaScript's safe integer range; export larger values as strings to preserve them exactly.`,
);
}
+export function requireJsonNumber(value: number, label: string): void {
+ if (!Number.isFinite(value))
+ throw new Error(`${label} must contain finite JSON numbers.`);
+ requireSafeJsonInteger(value, label);
+}
+
/** Report metadata is retained in previews and receipts without Cloud evidence size limits. */
export function validateExternalReport(
report: Record<string, unknown> | undefined,
): void {
if (report === undefined) return;
- JSON.stringify(report, (_key, value: unknown) => {
- if (isNumberObject(value)) value = +value;
- if (typeof value === "number") requireJsonNumber(value, "Scanner report");
- return value;
- });
+ serializeImportJson(report, checkReportNumber);
}
-export function validateExternalEvidence(
+export function validateExternalEvidenceFields(
input: unknown,
target: ExternalImportTarget = "cloud",
): ExternalFindingEvidence {
@@ -326,49 +355,344 @@ export function validateExternalEvidence(
"Evidence URLs must be HTTP(S) links without credentials.",
);
}
- validateExternalEvidenceJson(result, target);
return result;
}
+// Bind changed admission to the validated row or the error returned to its reader.
+const previousAdmissions = new WeakMap<
+ object,
+ ExternalFindingEvidence | null | "uncertain"
+>();
+
+export function previousAdmission(value: object) {
+ return previousAdmissions.get(value);
+}
+
+export function validateExternalEvidence(
+ input: unknown,
+ target: ExternalImportTarget = "cloud",
+): ExternalFindingEvidence {
+ const result = validateExternalEvidenceFields(input, target);
+ let changed = false;
+ try {
+ validateExternalEvidenceJson(result, target, 0, (previous, current) => {
+ changed =
+ previous > CLOUD_EVIDENCE_BYTES !== current > CLOUD_EVIDENCE_BYTES;
+ });
+ } catch (error) {
+ if (changed && error instanceof Error)
+ previousAdmissions.set(error, result);
+ throw error;
+ }
+ if (changed) previousAdmissions.set(result, null);
+ return result;
+}
+
+function cloudNumberByteAdjustment(value: number): number {
+ if (!value || Math.abs(value) >= 1e-4) return 0;
+ const wire = JSON.stringify(value);
+ const cloud = (wire.includes(".") ? value.toExponential() : wire).replace(
+ /e-(\d)$/u,
+ "e-0$1",
+ );
+ return cloud.length - wire.length;
+}
+
+type ValidateNumber = (value: number) => void;
+const evidenceNumber: ValidateNumber = (value) =>
+ requireJsonNumber(value, "Evidence");
+
+const rawJson = JSON as JSON & {
+ isRawJSON?(value: unknown): value is { rawJSON: string };
+};
+/** Check the numeric value JSON emits before parsing can round a raw decimal. */
+export function checkEmittedJsonNumber(
+ value: unknown,
+ check: ValidateNumber,
+): number | undefined {
+ if (typeof value === "number") {
+ check(value);
+ return value;
+ }
+ if (!rawJson.isRawJSON?.(value)) return;
+ const token: unknown = JSON.parse(value.rawJSON);
+ if (typeof token !== "number") return;
+ check(token);
+ if (canonicalNumber(value.rawJSON) !== String(token))
+ throw new Error(
+ "Raw JSON number loses precision when converted to a JavaScript number. Export the value as a string to preserve it exactly.",
+ );
+ return token;
+}
+
+const reportNumber: ValidateNumber = (value) =>
+ requireJsonNumber(value, "Scanner report");
+
+export function checkReportNumber(value: unknown): void {
+ checkEmittedJsonNumber(value, reportNumber);
+}
+
+export class CloudEvidenceDepthError extends Error {
+ constructor() {
+ super("Evidence exceeds the Cloud JSON depth limit of 20.");
+ }
+}
+
+/** JSON omits these object properties, but emits null for the same array elements. */
+export function requireCloudEvidenceDepth(
+ depth: number,
+ holder: object,
+ value: unknown,
+): void {
+ if (
+ depth > 20 &&
+ (Array.isArray(holder) ||
+ (value !== undefined &&
+ typeof value !== "symbol" &&
+ typeof value !== "function"))
+ )
+ throw new CloudEvidenceDepthError();
+}
+
+function serializeImportJson(
+ input: unknown,
+ validate: (value: unknown) => void,
+ limitDepth = false,
+): string {
+ const parents: object[] = [];
+ return JSON.stringify(input, function (this: object, _key, value: unknown) {
+ if (limitDepth) {
+ while (parents.length && parents.at(-1) !== this) parents.pop();
+ requireCloudEvidenceDepth(parents.length, this, value);
+ if (value !== null && typeof value === "object") parents.push(value);
+ }
+ // JSON applies Number conversion to wrappers after calling toJSON.
+ if (isNumberObject(value)) value = +value;
+ validate(value);
+ return value;
+ });
+}
+
+function* dataValues(value: object, skipped?: () => void): Generator<unknown> {
+ if (isProxy(value)) {
+ skipped?.();
+ return;
+ }
+ const keys = Object.keys(value);
+ // Array indices precede named keys. Dense arrays retain every index in a native clone.
+ if (
+ Array.isArray(value) &&
+ value.length &&
+ keys[value.length - 1] !== String(value.length - 1) &&
+ Object.getOwnPropertyNames(value).some(
+ (key) =>
+ /^(?:0|[1-9]\d*)$/u.test(key) &&
+ Number(key) < value.length &&
+ !Object.prototype.propertyIsEnumerable.call(value, key),
+ )
+ ) {
+ // The released mapper could read non-enumerable indices before cloning them away.
+ skipped?.();
+ }
+ for (const key of keys) {
+ const field = Object.getOwnPropertyDescriptor(value, key);
+ if (field && Object.hasOwn(field, "value")) yield field.value;
+ else skipped?.();
+ }
+}
+
+// Validate original numeric fields even if a serializer omits them. Object.values
+// reads siblings before descending, as the released provider reader did.
+function rawJsonNumbers(
+ input: unknown,
+ target: ExternalImportTarget,
+ oldSize = false,
+ dataOnly = false,
+ skipped?: () => void,
+): number {
+ let adjustment = 0;
+ function countNumber(value: number) {
+ evidenceNumber(value);
+ if (target === "cloud") adjustment += cloudNumberByteAdjustment(value);
+ }
+ function exceedDepth() {
+ if (!oldSize)
+ throw new Error("Evidence exceeds the Cloud JSON depth limit of 20.");
+ adjustment = Infinity;
+ }
+
+ if (input === null || typeof input !== "object") {
+ if (typeof input === "number") countNumber(input);
+ return adjustment;
+ }
+ const pending: object[] = [input];
+ const depths = [0];
+ const ancestors: object[] = [];
+ let checked = target === "local" ? new WeakSet<object>() : undefined;
+ while (pending.length) {
+ const value = pending.pop()!;
+ const depth = depths.pop()!;
+ if (target === "cloud" && depth > 20) exceedDepth();
+ // After a historical rejection, aliases only need one raw numeric check.
+ if (adjustment === Infinity) checked ??= new WeakSet();
+ if (checked?.has(value)) continue;
+ checked?.add(value);
+ ancestors.length = depth;
+ if (ancestors.includes(value)) {
+ if (oldSize) adjustment = Infinity;
+ continue;
+ }
+ ancestors[depth] = value;
+ for (const child of dataOnly
+ ? dataValues(value, skipped)
+ : Object.values(value)) {
+ if (typeof child === "number") countNumber(child);
+ if (child !== null && typeof child === "object") {
+ pending.push(child);
+ depths.push(depth + 1);
+ } else if (target === "cloud" && depth >= 20) exceedDepth();
+ }
+ }
+ return adjustment;
+}
+
+const CLOUD_EVIDENCE_BYTES = 256 * 1024;
+function checkEvidenceSize(bytes: number): number {
+ if (bytes > CLOUD_EVIDENCE_BYTES)
+ throw new Error("Evidence exceeds the Cloud limit of 256 KiB.");
+ return bytes;
+}
+
/** Return the validated Cloud wire size, or zero for local evidence. */
export function validateExternalEvidenceJson(
input: unknown,
target: ExternalImportTarget,
additionalCloudBytes = 0,
+ observe?: (previousBytes: number, currentBytes: number) => void,
): number {
- // Local evidence still has to be serializable JSON, including acyclic source data.
- if (target === "local") JSON.stringify(input);
- let numericByteAdjustment = 0;
- const pending: [unknown, number][] = [[input, 0]];
- while (pending.length > 0) {
- const [value, depth] = pending.pop()!;
- if (target === "cloud" && depth > 20)
- throw new Error("Evidence exceeds the Cloud JSON depth limit of 20.");
- if (value !== null && typeof value === "object") {
- for (const child of Object.values(value))
- pending.push([child, depth + 1]);
- } else if (typeof value === "number") {
- requireJsonNumber(value, "Evidence");
- if (target === "local") continue;
- const wire = JSON.stringify(value);
- // Cloud pads short exponents and formats small fractions scientifically.
- const cloud = (
- wire.includes(".") && Math.abs(value) < 1e-4
- ? value.toExponential()
- : wire
- ).replace(/e-(\d)$/u, "e-0$1");
- numericByteAdjustment += cloud.length - wire.length;
+ let adjustment = 0;
+ const validateNumber = (value: unknown) => {
+ try {
+ const number = checkEmittedJsonNumber(value, evidenceNumber);
+ if (target === "cloud" && number !== undefined)
+ adjustment += cloudNumberByteAdjustment(number);
+ } catch (error) {
+ // The released walk checked primitive numeric fields, not JSON-emitted wrappers.
+ if (error instanceof Error) previousAdmissions.set(error, "uncertain");
+ throw error;
+ }
+ };
+ // Keep local serialization before the raw walk. If an emitted number is invalid,
+ // ordinary invalid primitive fields still give a definitive historical exclusion.
+ if (target === "local") {
+ let invalid: Error | undefined;
+ try {
+ serializeImportJson(input, validateNumber);
+ } catch (error) {
+ if (!(error instanceof Error) || previousAdmission(error) !== "uncertain")
+ throw error;
+ invalid = error;
+ }
+ rawJsonNumbers(input, target);
+ if (invalid) throw invalid;
+ return 0;
+ }
+ const rawAdjustment = rawJsonNumbers(input, target);
+ let text: string;
+ try {
+ text = serializeImportJson(input, validateNumber, true);
+ } catch (error) {
+ // A serializer can introduce levels that were absent from the raw evidence.
+ if (error instanceof CloudEvidenceDepthError)
+ previousAdmissions.set(error, "uncertain");
+ throw error;
+ }
+ const bytes = Buffer.byteLength(text) + additionalCloudBytes;
+ observe?.(bytes + rawAdjustment, bytes + adjustment);
+ return checkEvidenceSize(bytes + adjustment);
+}
+
+/** Reproduce the released SDK's checks on a detached native finding. */
+export function validatePreviousCloudEvidence(
+ input: unknown,
+): ExternalFindingEvidence {
+ const evidence = validateExternalEvidenceFields(input, "cloud");
+ // Older validation counted enumerable primitive numbers, while JSON encoded
+ // the native wrappers. Check depth before serializing a possibly shared graph.
+ const adjustment = rawJsonNumbers(evidence, "cloud", true, true);
+ if (!Number.isFinite(adjustment)) throw new CloudEvidenceDepthError();
+ checkEvidenceSize(Buffer.byteLength(JSON.stringify(evidence)) + adjustment);
+ return evidence;
+}
+
+/** Use the array index as toJSON's key, as when saving the supplied GitHub records. */
+export function exportedGitHubRecord(
+ value: unknown,
+ index: number,
+ target: ExternalImportTarget,
+ checkPreviousLocal = false,
+) {
+ // The released readers validated numeric fields hidden by serializers.
+ let adjustment = 0;
+ let previousInvalid: unknown;
+ let skippedRawValue = false;
+ if (target === "cloud" || checkPreviousLocal) {
+ try {
+ adjustment = rawJsonNumbers(
+ { source_data: value },
+ target,
+ target === "cloud",
+ true,
+ () => {
+ skippedRawValue = true;
+ },
+ );
+ } catch (error) {
+ previousInvalid = error;
}
}
+ let invalid: unknown;
+ const text = serializeImportJson(
+ { [index]: value },
+ (value) => {
+ if (!invalid)
+ try {
+ checkEmittedJsonNumber(value, evidenceNumber);
+ } catch (error) {
+ invalid = error;
+ }
+ },
+ target === "cloud",
+ );
+ const record: unknown = JSON.parse(text)[index];
const bytes =
- target === "cloud"
- ? Buffer.byteLength(JSON.stringify(input)) +
- numericByteAdjustment +
- additionalCloudBytes
- : 0;
- if (bytes > 256 * 1024)
- throw new Error("Evidence exceeds the Cloud limit of 256 KiB.");
- return bytes;
+ Buffer.byteLength(text) -
+ Buffer.byteLength(JSON.stringify({ [index]: null })) +
+ 4;
+ return {
+ record,
+ bytes,
+ adjustment,
+ previousInvalid,
+ skippedRawValue,
+ invalid,
+ };
+}
+
+/** Combine the observed vendor bytes with the old native mapping's typed fields. */
+export function previousGitHubExclusion(
+ evidence: ExternalFindingEvidence,
+ sourceBytes: number,
+ adjustment: number,
+): string | undefined {
+ if (adjustment === Infinity)
+ return "Evidence exceeds the Cloud JSON depth limit of 20.";
+ const envelope = { ...evidence, source_data: null };
+ const bytes = Buffer.byteLength(JSON.stringify(envelope)) - 4 + sourceBytes;
+ return bytes + adjustment + rawJsonNumbers(envelope, "cloud", true) >
+ CLOUD_EVIDENCE_BYTES
+ ? "Evidence exceeds the Cloud limit of 256 KiB."
+ : undefined;
}
export function validateImportRequest(input: unknown): FindingImportRequest {
diff --git a/sdk/typescript/src/github-dependabot-findings.ts b/sdk/typescript/src/github-dependabot-findings.ts
index 533606cf9..04af0f387 100644
--- a/sdk/typescript/src/github-dependabot-findings.ts
+++ b/sdk/typescript/src/github-dependabot-findings.ts
@@ -1,6 +1,6 @@
import { z } from "zod";
import { GHSA_ID, githubRepositoryParts } from "./github-identifiers.js";
-import { CodexSecurityError, errorMessage } from "./errors.js";
+import { CodexSecurityError } from "./errors.js";
import {
type ExternalImportTarget,
validateExternalEvidence,
@@ -8,7 +8,13 @@ import {
import type { ExternalFinding, ExternalFindings } from "./external-findings.js";
import { isGitHubCloudHost } from "./git-remote.js";
import { isRecord } from "./record.js";
-import { record, relativePath, requiredText, text } from "./scanner-import.js";
+import {
+ trackReaderSelection,
+ record,
+ relativePath,
+ requiredText,
+ text,
+} from "./scanner-import.js";
export function isGitHubDependabotAlert(
value: unknown,
@@ -313,6 +319,7 @@ export function normalizeGitHubDependabotAlerts(
findings: [],
excluded: [],
};
+ const selection = trackReaderSelection(result);
const ids = new Set<string>();
let sourceRepository: string | undefined;
for (const [index, alert] of alerts.entries()) {
@@ -328,22 +335,13 @@ export function normalizeGitHubDependabotAlerts(
);
ids.add(sourceId);
const inputEvidence = evidence(alert, url);
- try {
+ selection.append(index + 1, sourceId, () => {
if (target === "cloud" && Buffer.byteLength(sourceId) > 512)
throw new Error(
"Source finding ID exceeds the Cloud limit of 512 UTF-8 bytes.",
);
- result.findings.push({
- source_finding_id: sourceId,
- evidence: validateExternalEvidence(inputEvidence, target),
- });
- } catch (error) {
- result.excluded.push({
- position: index + 1,
- source_finding_id: sourceId,
- reason: errorMessage(error),
- });
- }
+ return validateExternalEvidence(inputEvidence, target);
+ });
}
- return result;
+ return selection.finish();
}
diff --git a/sdk/typescript/src/github-findings.ts b/sdk/typescript/src/github-findings.ts
index e4d1232d5..ced61f8d1 100644
--- a/sdk/typescript/src/github-findings.ts
+++ b/sdk/typescript/src/github-findings.ts
@@ -1,8 +1,17 @@
import type { ExternalImportTarget } from "./external-import-contract.js";
import { hash } from "node:crypto";
+import { isDeepStrictEqual } from "node:util";
+import { isBoxedPrimitive, isDate, isProxy } from "node:util/types";
import { GHSA_ID, githubRepositoryParts } from "./github-identifiers.js";
import { CodexSecurityError, errorMessage } from "./errors.js";
-import { validateExternalEvidence } from "./external-import-contract.js";
+import {
+ CloudEvidenceDepthError,
+ exportedGitHubRecord,
+ previousGitHubExclusion,
+ validateExternalEvidenceFields,
+ validateExternalEvidenceJson,
+ validateExternalEvidence,
+} from "./external-import-contract.js";
import type { ExternalFinding, ExternalFindings } from "./external-findings.js";
import type { ImportedGitHubCodeScanningAlert } from "./github.js";
import type { ImportedGitHubAdvisory } from "./github-advisories.js";
@@ -10,9 +19,18 @@ import {
isGitHubDependabotAlert,
normalizeGitHubDependabotAlerts,
} from "./github-dependabot-findings.js";
-import { asRecord } from "./record.js";
+import {
+ asRecord,
+ canSnapshotWithoutGetters,
+ cloneWithoutGetters,
+} from "./record.js";
import { parseJsonOrJsonl, readScannerText, text } from "./scanner-import.js";
+import {
+ rememberPreviousSelection,
+ type LegacyReaderSelection,
+} from "./reader-selection.js";
+
function repositoryName(value: unknown): string {
const repository = text(value);
if (!repository || !githubRepositoryParts(repository))
@@ -48,10 +66,22 @@ function severity(value: unknown): string {
: "unknown";
}
-function codeScanningFinding(
- record: Record<string, unknown>,
- target: ExternalImportTarget = "cloud",
-): ExternalFinding {
+function codeScanningPrefix(
+ { parsed, host }: ReturnType<typeof sourceUrl>,
+ repository: string,
+ number: number,
+): string | undefined {
+ const name = repository.toLowerCase();
+ const path = parsed.pathname.toLowerCase();
+ const alert = `code-scanning/${number}`;
+ if (path === `/${name}/${alert}` || path === `/${name}/security/${alert}`)
+ return `${host}/${name}/${alert}/`;
+}
+
+function codeScanningFinding(record: Record<string, unknown>): {
+ source_finding_id: string;
+ evidence: unknown;
+} {
const repository = repositoryName(record["repository"]);
const alert = asRecord(record["alert"]);
const number = record["number"];
@@ -68,14 +98,9 @@ function codeScanningFinding(
const url = text(record["url"]);
if (!url || url !== alert["html_url"])
throw new Error("GitHub alert URL must match the original alert.");
- const { parsed: parsedUrl, host, origin } = sourceUrl(url);
- if (
- !["security/code-scanning", "code-scanning"].some(
- (route) =>
- parsedUrl.pathname.toLowerCase() ===
- `/${repository}/${route}/${number}`.toLowerCase(),
- )
- )
+ const link = sourceUrl(url);
+ const prefix = codeScanningPrefix(link, repository, number);
+ if (!prefix)
throw new Error(
"GitHub alert URL does not match the selected repository and alert.",
);
@@ -104,54 +129,57 @@ function codeScanningFinding(
),
];
const path = text(location?.["path"]);
- const sourceId = `${host}/${repository.toLowerCase()}/code-scanning/${number}/${hash("sha256", ref)}`;
return {
- source_finding_id: sourceId,
- evidence: validateExternalEvidence(
- {
- title:
- text(rule?.["name"]) ??
- text(rule?.["description"]) ??
- `GitHub code-scanning alert #${number}`,
- severity: severity(rule?.["security_severity_level"]),
- description:
- [...new Set([message, description].filter(Boolean))].join("\n\n") ||
- null,
- url,
- locations: path
- ? [{ path, line: location?.["start_line"] ?? null }]
- : [],
- branch: ref,
- code_revision: text(instance?.["commit_sha"]),
- source_updated_at: sourceTimestamp(alert["updated_at"]),
- source_data: record,
- details: {
- kind: "sast",
- repository: {
- url: `${origin}/${repository}`,
- name: repository,
- },
- // Dismissal covers every branch; detection state belongs to the selected ref.
- source_status:
- alert["state"] === "dismissed"
- ? "dismissed"
- : (text(instance?.["state"]) ?? text(alert["state"])),
- scanner_origin: text(tool?.["name"]),
- rule: { id: text(rule?.["id"]), name: text(rule?.["name"]) },
- weakness_ids: weaknessIds,
- remediation_instructions: text(rule?.["help"]),
- code: path ? { end_line: location?.["end_line"] ?? null } : null,
+ source_finding_id: prefix + hash("sha256", ref),
+ evidence: {
+ title:
+ text(rule?.["name"]) ??
+ text(rule?.["description"]) ??
+ `GitHub code-scanning alert #${number}`,
+ severity: severity(rule?.["security_severity_level"]),
+ description:
+ [...new Set([message, description].filter(Boolean))].join("\n\n") ||
+ null,
+ url,
+ locations: path ? [{ path, line: location?.["start_line"] ?? null }] : [],
+ branch: ref,
+ code_revision: text(instance?.["commit_sha"]),
+ source_updated_at: sourceTimestamp(alert["updated_at"]),
+ source_data: record,
+ details: {
+ kind: "sast",
+ repository: {
+ url: `${link.origin}/${repository}`,
+ name: repository,
},
+ // Dismissal covers every branch; detection state belongs to the selected ref.
+ source_status:
+ alert["state"] === "dismissed"
+ ? "dismissed"
+ : (text(instance?.["state"]) ?? text(alert["state"])),
+ scanner_origin: text(tool?.["name"]),
+ rule: { id: text(rule?.["id"]), name: text(rule?.["name"]) },
+ weakness_ids: weaknessIds,
+ remediation_instructions: text(rule?.["help"]),
+ code: path ? { end_line: location?.["end_line"] ?? null } : null,
},
- target,
- ),
+ },
};
}
-function advisoryFinding(
- record: Record<string, unknown>,
- target: ExternalImportTarget = "cloud",
-): ExternalFinding {
+function advisorySourceId(
+ host: string,
+ selection: "global" | "repository",
+ repository: string,
+ id: string,
+): string {
+ return `${host}/${selection === "repository" ? repository.toLowerCase() : "global"}/security-advisory/${id.toUpperCase()}`;
+}
+
+function advisoryFinding(record: Record<string, unknown>): {
+ source_finding_id: string;
+ evidence: unknown;
+} {
const repository = repositoryName(record["repository"]);
const advisory = asRecord(record["advisory"]);
const id = text(record["ghsaId"]);
@@ -222,79 +250,402 @@ function advisoryFinding(
});
const cwes = Array.isArray(advisory["cwes"]) ? advisory["cwes"] : [];
return {
- source_finding_id: `${host}/${selection === "repository" ? repository.toLowerCase() : "global"}/security-advisory/${id.toUpperCase()}`,
- evidence: validateExternalEvidence(
- {
- title: text(advisory["summary"]) ?? id,
- severity: severity(advisory["severity"]),
- description: text(advisory["description"]),
- url,
- advisory_ids: [...new Set(advisoryIds)],
- packages,
- source_updated_at: sourceTimestamp(advisory["updated_at"]),
- source_data: record,
- details: {
- kind: "advisory",
- advisory: { source: selection },
- ...(selection === "repository"
- ? {
- repository: {
- url: `${origin}/${repository}`,
- name: repository,
- },
- }
- : {}),
- source_status: text(advisory["withdrawn_at"])
- ? "withdrawn"
- : (text(advisory["state"]) ??
- (selection === "global" ? "published" : null)),
- scanner_origin: "GitHub Security Advisories",
- weakness_ids: cwes
- .map((value) => text(asRecord(value)?.["cwe_id"]))
- .filter((value): value is string => value !== null),
- },
+ source_finding_id: advisorySourceId(host, selection, repository, id),
+ evidence: {
+ title: text(advisory["summary"]) ?? id,
+ severity: severity(advisory["severity"]),
+ description: text(advisory["description"]),
+ url,
+ advisory_ids: [...new Set(advisoryIds)],
+ packages,
+ source_updated_at: sourceTimestamp(advisory["updated_at"]),
+ source_data: record,
+ details: {
+ kind: "advisory",
+ advisory: { source: selection },
+ ...(selection === "repository"
+ ? {
+ repository: {
+ url: `${origin}/${repository}`,
+ name: repository,
+ },
+ }
+ : {}),
+ source_status: text(advisory["withdrawn_at"])
+ ? "withdrawn"
+ : (text(advisory["state"]) ??
+ (selection === "global" ? "published" : null)),
+ scanner_origin: "GitHub Security Advisories",
+ weakness_ids: cwes
+ .map((value) => text(asRecord(value)?.["cwe_id"]))
+ .filter((value): value is string => value !== null),
},
- target,
- ),
+ },
+ };
+}
+
+function mapRecord(input: unknown) {
+ const record = asRecord(input);
+ if (!record) throw new Error("GitHub finding must be an object.");
+ if (record["source"] === "github-code-scanning")
+ return codeScanningFinding(record);
+ if (record["source"] === "github-security-advisory")
+ return advisoryFinding(record);
+ throw new Error(
+ "Expected saved output from import github or import github-advisories; raw or incomplete API responses need an explicit mapping.",
+ );
+}
+
+function ownData(value: unknown, key: string): unknown {
+ if (value && typeof value === "object" && !isProxy(value))
+ return Object.getOwnPropertyDescriptor(value, key)?.value;
+}
+
+function priorAdvisoryId(value: unknown): string | undefined {
+ if (ownData(value, "source") !== "github-security-advisory") return;
+ const id = text(ownData(value, "ghsaId"));
+ const host = text(ownData(value, "githubHost"))?.toLowerCase();
+ const selection = ownData(value, "selection");
+ const repository = text(ownData(value, "repository"));
+ if (
+ id &&
+ GHSA_ID.test(id) &&
+ host &&
+ repository &&
+ githubRepositoryParts(repository) &&
+ (selection === "repository" || selection === "global")
+ )
+ return advisorySourceId(host, selection, repository, id);
+}
+
+// Determine whether JSON can call a serializer without evaluating a getter.
+function maySerialize(value: unknown): boolean {
+ while (value && typeof value === "object") {
+ if (isProxy(value)) return true;
+ const field = Object.getOwnPropertyDescriptor(value, "toJSON");
+ if (field)
+ return (
+ !Object.hasOwn(field, "value") || typeof field.value === "function"
+ );
+ value = Object.getPrototypeOf(value) as unknown;
+ }
+ return false;
+}
+
+function codeOccurrencePrefix(value: unknown): string | undefined {
+ const repository = ownData(value, "repository"),
+ number = ownData(value, "number"),
+ url = ownData(value, "url");
+ if (
+ ownData(value, "source") !== "github-code-scanning" ||
+ typeof repository !== "string" ||
+ !githubRepositoryParts(repository) ||
+ typeof number !== "number" ||
+ !Number.isSafeInteger(number) ||
+ number <= 0 ||
+ typeof url !== "string"
+ )
+ return;
+ try {
+ return codeScanningPrefix(sourceUrl(url), repository, number);
+ } catch {
+ /* No validated occurrence anchor. */
+ }
+}
+
+// The released reader measured the caller's source_data JSON before native capture.
+function captureGitHubRecord(input: unknown, target: ExternalImportTarget) {
+ // Only plain records, arrays and standard Buffers have a reconstructable wire size.
+ let plainBuffers: Buffer[] | undefined = [];
+ let hiddenFields = false;
+ let largeLocalRow = false;
+ const record = cloneWithoutGetters(input, (value) => {
+ const prototype = Object.getPrototypeOf(value);
+ if (
+ target === "local" &&
+ ((Array.isArray(value) && value.length > 256 * 1024) ||
+ (prototype === Buffer.prototype &&
+ Buffer.isBuffer(value) &&
+ !Object.hasOwn(value, "byteLength") &&
+ value.byteLength > 256 * 1024))
+ ) {
+ largeLocalRow = true;
+ return false;
+ }
+ // Native capture drops non-enumerable fields the released vendor mapper could read.
+ if (
+ !Array.isArray(value) &&
+ !ArrayBuffer.isView(value) &&
+ !isBoxedPrimitive(value) &&
+ !isDate(value) &&
+ Object.getOwnPropertyNames(value).some(
+ (key) =>
+ key !== "toJSON" &&
+ !Object.prototype.propertyIsEnumerable.call(value, key),
+ )
+ )
+ hiddenFields = true;
+
+ if (!plainBuffers) return;
+ if (Buffer.isBuffer(value)) {
+ if (
+ prototype === Buffer.prototype &&
+ !Object.hasOwn(value, "toJSON") &&
+ !Object.hasOwn(value, "length") &&
+ !("_arr" in value)
+ )
+ plainBuffers.push(value);
+ else plainBuffers = undefined;
+ } else if (
+ (prototype !== Object.prototype &&
+ prototype !== null &&
+ prototype !== Array.prototype) ||
+ maySerialize(value)
+ )
+ plainBuffers = undefined;
+ });
+ return {
+ record: asRecord(record),
+ hiddenFields,
+ plainBuffers,
+ largeLocalRow,
};
}
function normalizeRecords(
records: readonly unknown[],
target: ExternalImportTarget = "cloud",
+ snapshot = false,
): ExternalFindings {
const result: ExternalFindings = {
read: records.length,
findings: [],
excluded: [],
};
+ const previous: ExternalFindings | undefined = snapshot
+ ? { read: records.length, findings: [], excluded: [] }
+ : undefined;
+ const checks: (() => void)[] = [];
+ const uncertainIds = new Set<string>();
+ const unidentifiedRows = new Set<number>();
+ let independentlyObserved = Infinity;
+ let ranRootSerializer = false;
+ let largeLocalSelection = false;
+ const potentialIds: string[] = [];
+ const prefixes = new Set<string>();
const identities = new Set<string>();
- for (const [index, value] of records.entries()) {
- const record = asRecord(value);
+ for (const [index, input] of records.entries()) {
+ const afterRootSerializer = ranRootSerializer;
+ const keySensitive = previous !== undefined && maySerialize(input);
+ ranRootSerializer ||= keySensitive;
+ let exported = input;
+ const capture = previous ? captureGitHubRecord(input, target) : undefined;
+ const before = capture?.record;
+ const priorPrefix = previous ? codeOccurrencePrefix(input) : undefined;
+ let priorId = previous ? priorAdvisoryId(input) : undefined;
+ let prefix = priorPrefix;
+ const rowIds = new Set<string>();
+ let rowUncertain = false;
+ const priorRowCount = previous?.findings.length ?? 0;
+ function possible() {
+ rowUncertain = true;
+ // A caller getter can mutate a later row before its native evidence is seen.
+ independentlyObserved = Math.min(independentlyObserved, priorRowCount);
+ for (const id of rowIds) uncertainIds.add(id);
+ if (prefix) prefixes.add(prefix);
+ if (priorId) uncertainIds.add(priorId);
+ else if (!priorPrefix) unidentifiedRows.add(index);
+ }
+ // The old reader called a root serializer with "source_data" rather than its
+ // array index. That code could change a later occurrence before it was read.
+ if (afterRootSerializer) {
+ possible();
+ unidentifiedRows.add(index);
+ }
+ function retainPrevious(
+ selection: ExternalFindings,
+ id: string,
+ evidence: ExternalFinding["evidence"],
+ ) {
+ selection.findings.push({ source_finding_id: id, evidence });
+ potentialIds.push(id);
+ checks.push(() => {
+ if (!isDeepStrictEqual(before, asRecord(cloneWithoutGetters(input))))
+ possible();
+ });
+ }
+ const exclusion = (record: unknown, reason: string) => ({
+ position: index + 1,
+ source_finding_id: text(asRecord(record)?.["url"]),
+ reason,
+ });
+ let possibleFreshId = false;
try {
- if (!record) throw new Error("GitHub finding must be an object.");
- let finding: ExternalFinding;
- if (record["source"] === "github-code-scanning")
- finding = codeScanningFinding(record, target);
- else if (record["source"] === "github-security-advisory")
- finding = advisoryFinding(record, target);
- else
- throw new Error(
- "Expected saved output from import github or import github-advisories; raw or incomplete API responses need an explicit mapping.",
+ if (previous && capture) {
+ const source = exportedGitHubRecord(
+ input,
+ index,
+ target,
+ target === "local" && !capture.largeLocalRow,
);
- if (identities.has(finding.source_finding_id))
+ exported = source.record;
+ prefix ??= codeOccurrencePrefix(exported);
+ // A serializer or Buffer can expand a native record that fit the approved request.
+ const oversizedLocalRow =
+ target === "local" &&
+ (capture.largeLocalRow ||
+ (source.bytes > 256 * 1024 &&
+ (!before || capture.plainBuffers?.length === 0)));
+ if (oversizedLocalRow || !before) {
+ if (
+ !oversizedLocalRow &&
+ source.previousInvalid &&
+ !source.skippedRawValue &&
+ canSnapshotWithoutGetters(input)
+ ) {
+ previous.excluded.push(
+ exclusion(exported, errorMessage(source.previousInvalid)),
+ );
+ } else if (
+ target !== "local" &&
+ source.bytes + source.adjustment > 256 * 1024 &&
+ canSnapshotWithoutGetters(input) &&
+ !keySensitive
+ ) {
+ previous.excluded.push(
+ exclusion(
+ exported,
+ "Evidence exceeds the Cloud limit of 256 KiB.",
+ ),
+ );
+ } else {
+ largeLocalSelection ||= oversizedLocalRow;
+ // An occurrence without independent native evidence can only stop a retry.
+ possible();
+ possibleFreshId = true;
+ }
+ } else {
+ if (capture.hiddenFields || source.skippedRawValue) possible();
+ try {
+ const row = mapRecord(before);
+ const knownOldId = row.source_finding_id;
+ priorId = knownOldId;
+ if (!afterRootSerializer) unidentifiedRows.delete(index);
+ rowIds.add(knownOldId);
+ if (rowUncertain) uncertainIds.add(knownOldId);
+ const evidence = validateExternalEvidenceFields(
+ row.evidence,
+ target,
+ );
+ if (source.previousInvalid) throw source.previousInvalid;
+ if (keySensitive) possible();
+ const reason =
+ keySensitive || target === "local"
+ ? undefined
+ : previousGitHubExclusion(
+ evidence,
+ source.bytes,
+ source.adjustment,
+ );
+ if (reason) previous.excluded.push(exclusion(before, reason));
+ else retainPrevious(previous, knownOldId, evidence);
+ } catch (error) {
+ previous.excluded.push(exclusion(before, errorMessage(error)));
+ }
+ }
+ if (source.invalid) throw source.invalid;
+ }
+ const mapped = mapRecord(exported),
+ id = mapped.source_finding_id;
+ rowIds.add(id);
+ if (rowUncertain) uncertainIds.add(id);
+ if (possibleFreshId) potentialIds.push(id);
+ const evidence = validateExternalEvidence(mapped.evidence, target);
+ if (identities.has(id))
throw new CodexSecurityError(
- `Duplicate source finding id ${JSON.stringify(finding.source_finding_id)}. Select each GitHub occurrence once.`,
+ `Duplicate source finding id ${JSON.stringify(id)}. Select each GitHub occurrence once.`,
);
- identities.add(finding.source_finding_id);
- result.findings.push(finding);
+ identities.add(id);
+ result.findings.push({ source_finding_id: id, evidence });
} catch (error) {
if (error instanceof CodexSecurityError) throw error;
- result.excluded.push({
- position: index + 1,
- source_finding_id: text(record?.["url"]),
- reason: errorMessage(error),
- });
+ const reason = errorMessage(error);
+ result.excluded.push(
+ exclusion(
+ snapshot && exported === input
+ ? { url: ownData(input, "url") }
+ : exported,
+ reason,
+ ),
+ );
+ if (previous && exported === input) {
+ if (
+ error instanceof CloudEvidenceDepthError &&
+ before &&
+ // Extra Buffer fields were inspected by the old row validator, but
+ // structuredClone does not retain them as part of the typed array.
+ capture?.plainBuffers?.every(
+ (buffer) => Object.keys(buffer).length === buffer.length,
+ )
+ ) {
+ // Exported JSON can be deeper than an approved native snapshot.
+ // Retain only the independently observed, valid prior selection.
+ try {
+ const row = mapRecord(before);
+ const evidence = validateExternalEvidence(row.evidence, target);
+ const savedSource = JSON.stringify(evidence.source_data);
+ // structuredClone turns each Buffer into a Uint8Array. Reconstruct its
+ // original serialized size on detached data without running caller hooks.
+ const oldSource = JSON.stringify(
+ evidence.source_data,
+ (_key, value: unknown) =>
+ value instanceof Uint8Array
+ ? { type: "Buffer", data: [...value] }
+ : value,
+ );
+ validateExternalEvidenceJson(
+ evidence,
+ target,
+ Buffer.byteLength(oldSource) - Buffer.byteLength(savedSource),
+ );
+ retainPrevious(previous, row.source_finding_id, evidence);
+ } catch (priorError) {
+ previous.excluded.push(exclusion(before, errorMessage(priorError)));
+ }
+ continue;
+ }
+ // A serializer may throw at its array key while the old source_data
+ // key produced a valid record. Keep only evidence for refusing a match.
+ possible();
+ if (before)
+ try {
+ const id = mapRecord(before).source_finding_id;
+ priorId = id;
+ if (!afterRootSerializer) unidentifiedRows.delete(index);
+ potentialIds.push(id);
+ uncertainIds.add(id);
+ } catch {
+ /* No old identity. */
+ }
+ }
+ }
+ }
+ if (previous) {
+ for (const check of checks) check();
+ const uncertain = independentlyObserved !== Infinity;
+ if (uncertain || !isDeepStrictEqual(previous.findings, result.findings)) {
+ const prior: LegacyReaderSelection = { sourceFindingIds: potentialIds };
+ if (uncertain) {
+ prior.unprovenIdPrefixes = [...prefixes];
+ prior.uncertainFindingIds = [...uncertainIds];
+ if (unidentifiedRows.size)
+ prior.unprovenOccurrenceCount = unidentifiedRows.size;
+ if (!largeLocalSelection)
+ prior.knownFindings = previous.findings
+ .slice(0, independentlyObserved)
+ .filter((finding) => !uncertainIds.has(finding.source_finding_id));
+ } else prior.snapshot = previous;
+ rememberPreviousSelection(result, prior);
}
}
return result;
@@ -305,7 +656,7 @@ export function normalizeGitHubCodeScanningAlerts(
alerts: readonly ImportedGitHubCodeScanningAlert[],
target: ExternalImportTarget = "cloud",
): ExternalFindings {
- return normalizeRecords(alerts, target);
+ return normalizeRecords(alerts, target, true);
}
/** Map advisory disclosures without asserting an installed vulnerable dependency. */
@@ -313,7 +664,7 @@ export function normalizeGitHubAdvisories(
advisories: readonly ImportedGitHubAdvisory[],
target: ExternalImportTarget = "cloud",
): ExternalFindings {
- return normalizeRecords(advisories, target);
+ return normalizeRecords(advisories, target, true);
}
/** Read unfiltered JSON/JSONL output saved by the read-only GitHub import commands. */
diff --git a/sdk/typescript/src/inspector-findings.ts b/sdk/typescript/src/inspector-findings.ts
index b46a31c5a..efda61b11 100644
--- a/sdk/typescript/src/inspector-findings.ts
+++ b/sdk/typescript/src/inspector-findings.ts
@@ -671,13 +671,11 @@ export function normalizeInspectorFindings(
claims: repositoryClaims(finding),
}));
checkReportScope(rows, target, projectMap);
- const result = normalizeRecords(
- rows,
- ({ finding }, id) => evidence(selectedFinding(finding), id, source, target),
- ({ finding }) => identity(record(finding, "Inspector finding")).arn,
- target,
- );
- if (result.excluded.length && result.findings.length) {
+ function checkRetainedScope(
+ result: ExternalFindings,
+ removedIds?: Set<string>,
+ ) {
+ if (!result.excluded.length || !result.findings.length) return;
// Only surviving rows can supply shared provider and host evidence on replay.
const skipped = new Set(result.excluded.map((row) => row.position));
const retained = rows.filter((row) => !skipped.has(row.position));
@@ -690,12 +688,20 @@ export function normalizeInspectorFindings(
reason: errorMessage(error),
});
});
- result.findings = result.findings.filter(
- (_, i) => !skipped.has(retained[i]!.position),
- );
+ result.findings = result.findings.filter((finding, i) => {
+ if (!skipped.has(retained[i]!.position)) return true;
+ removedIds?.add(finding.source_finding_id);
+ return false;
+ });
result.excluded.sort((a, b) => a.position - b.position);
}
- return result;
+ return normalizeRecords(
+ rows,
+ ({ finding }, id) => evidence(selectedFinding(finding), id, source, target),
+ ({ finding }) => identity(record(finding, "Inspector finding")).arn,
+ target,
+ { filter: checkRetainedScope },
+ );
}
export async function readInspectorFindings(
diff --git a/sdk/typescript/src/opengrep-findings.ts b/sdk/typescript/src/opengrep-findings.ts
index 60ee33f06..c0de28b01 100644
--- a/sdk/typescript/src/opengrep-findings.ts
+++ b/sdk/typescript/src/opengrep-findings.ts
@@ -1,5 +1,6 @@
import { posix } from "node:path";
import {
+ previousAdmission,
validateExternalEvidenceJson,
validateExternalReport,
type ExternalImportTarget,
@@ -14,6 +15,7 @@ import {
sourceRepositoryUrl,
} from "./native-json-findings.js";
import {
+ trackReaderSelection,
array,
readScannerJson,
record,
@@ -57,11 +59,14 @@ export function normalizeOpenGrepFindings(
excluded: [],
report: structuredClone(report),
};
+ const selection = trackReaderSelection(output);
type Observation = {
position: number;
parsed: ReturnType<typeof parseNativeJsonResult>;
};
function exclude(position: number, id: string | null, error: unknown) {
+ if (id && error instanceof Error && previousAdmission(error) !== undefined)
+ selection.uncertainIds.add(id);
output.excluded.push({
position,
source_finding_id: id,
@@ -132,6 +137,8 @@ export function normalizeOpenGrepFindings(
projectedBytes,
);
} catch (error) {
+ if (error instanceof Error && previousAdmission(error) !== undefined)
+ selection.uncertainIds.add(id);
singletonErrors.set(observation, errorMessage(error));
}
}
@@ -191,28 +198,38 @@ export function normalizeOpenGrepFindings(
for (const [id, observations] of [...groups].sort(([left], [right]) =>
left < right ? -1 : left > right ? 1 : 0,
)) {
- const ordered =
- observations.length === 1
- ? observations
- : observations
- .flatMap((observation) => {
- try {
- const result = structuredClone(observation.parsed.result);
- validateExternalEvidenceJson(
- { source_data: { results: [result] } },
- target,
- );
- observation.parsed.result = result;
- return [{ observation, key: canonicalJson(result) }];
- } catch (error) {
- exclude(observation.position, id, error);
- return [];
- }
- })
- .sort((left, right) =>
- left.key < right.key ? -1 : left.key > right.key ? 1 : 0,
- )
- .map(({ observation }) => observation);
+ if (observations.length === 1) {
+ selection.append(observations[0]!.position, id, () =>
+ projectGroup(observations),
+ );
+ continue;
+ }
+ const ordered = observations
+ .flatMap((observation) => {
+ try {
+ const result = structuredClone(observation.parsed.result);
+ // The group adds these same raw bytes when choosing a representative
+ // and deciding which traces fit alongside it.
+ validateExternalEvidenceJson(
+ { source_data: { results: [result] } },
+ target,
+ 0,
+ (previousBytes, currentBytes) => {
+ if (previousBytes !== currentBytes)
+ selection.uncertainIds.add(id);
+ },
+ );
+ observation.parsed.result = result;
+ return [{ observation, key: canonicalJson(result) }];
+ } catch (error) {
+ exclude(observation.position, id, error);
+ return [];
+ }
+ })
+ .sort((left, right) =>
+ left.key < right.key ? -1 : left.key > right.key ? 1 : 0,
+ )
+ .map(({ observation }) => observation);
if (!ordered.length) continue;
let retained = ordered;
try {
@@ -226,6 +243,7 @@ export function normalizeOpenGrepFindings(
if (retained.length === ordered.length) throw error;
evidence = projectGroup(retained);
}
+ if (previousAdmission(evidence) === null) selection.uncertainIds.add(id);
output.findings.push({ source_finding_id: id, evidence });
} catch (error) {
for (const observation of retained)
@@ -233,7 +251,7 @@ export function normalizeOpenGrepFindings(
}
}
output.excluded.sort((left, right) => left.position - right.position);
- return output;
+ return selection.finish();
}
export async function readOpenGrepFindings(
diff --git a/sdk/typescript/src/reader-selection.ts b/sdk/typescript/src/reader-selection.ts
new file mode 100644
index 000000000..43b3e9521
--- /dev/null
+++ b/sdk/typescript/src/reader-selection.ts
@@ -0,0 +1,118 @@
+import { createHash } from "node:crypto";
+import { isDeepStrictEqual } from "node:util";
+import { isBoxedPrimitive, isProxy } from "node:util/types";
+import type { ExternalFindings } from "./external-findings.js";
+import { canonicalJson, cloneWithoutGetters } from "./record.js";
+
+export interface LegacyReaderSelection {
+ sourceFindingIds: string[];
+ snapshot?: ExternalFindings;
+ unprovenIdPrefixes?: string[];
+ uncertainFindingIds?: string[];
+ unprovenOccurrenceCount?: number;
+ knownFindings?: ExternalFindings["findings"];
+}
+const previousSelections = new WeakMap<
+ object,
+ {
+ exported?: string;
+ native?: ExternalFindings;
+ previous: LegacyReaderSelection;
+ }
+>();
+
+// The registry digest is process-local. Frame both ordered lists separately so a
+// local import never needs one JSON string containing all its findings.
+function exportedSelectionDigest(
+ result: ExternalFindings,
+ preserveScalar?: (value: object) => boolean,
+): string {
+ const { findings, excluded, ...metadata } = result;
+ const digest = createHash("sha256");
+ digest.update(canonicalJson(metadata, undefined, preserveScalar));
+ for (const rows of [excluded, findings]) {
+ digest.update("[");
+ for (let index = 0; index < rows.length; index++) {
+ // Keep the array index supplied to toJSON. An omitted entry in an array
+ // is JSON null, just like an explicit null or a hole.
+ const entry = canonicalJson(
+ { [index]: rows[index] },
+ undefined,
+ preserveScalar,
+ );
+ digest.update(entry === "{}" ? `{"${index}":null}` : entry);
+ }
+ digest.update("]");
+ }
+ return digest.digest("hex");
+}
+
+export function rememberPreviousSelection(
+ result: ExternalFindings,
+ prior: LegacyReaderSelection,
+): void {
+ // Stop-only hints do not require hashing unbounded local evidence.
+ const exported =
+ prior.snapshot || prior.knownFindings
+ ? exportedSelectionDigest(result, isBoxedPrimitive)
+ : undefined;
+ const selection = {
+ exported,
+ native: exported ? cloneWithoutGetters(result) : undefined,
+ previous: prior,
+ };
+ previousSelections.set(result, selection);
+ previousSelections.set(result.findings, selection);
+}
+
+/** Only unchanged SDK output can identify an older, differently serialized selection. */
+export function previousReaderSelection(
+ input: ExternalFindings,
+ current: ExternalFindings,
+ native?: ExternalFindings | null,
+): LegacyReaderSelection | undefined {
+ const selection =
+ previousSelections.get(input) ??
+ (!isProxy(input) &&
+ previousSelections.get(
+ Object.getOwnPropertyDescriptor(input, "findings")?.value,
+ ));
+ if (!selection) return undefined;
+ const observed = native === undefined ? cloneWithoutGetters(input) : native;
+ if (selection.native && observed && selection.exported) {
+ // Callers can attach a report after normalizing rows. Historical validation
+ // and the saved approval still bind the independently captured report.
+ const attachedReport =
+ !Object.hasOwn(selection.native, "report") &&
+ !Object.hasOwn(selection.previous.snapshot ?? {}, "report") &&
+ observed.report !== undefined;
+ const rows = (value: ExternalFindings) => {
+ if (!attachedReport) return value;
+ const { report: _report, ...readerOutput } = value;
+ return readerOutput;
+ };
+ if (
+ isDeepStrictEqual(selection.native, rows(observed)) &&
+ exportedSelectionDigest(rows(current)) === selection.exported
+ ) {
+ const prior = selection.previous;
+ return attachedReport && prior.snapshot
+ ? { ...prior, snapshot: { ...prior.snapshot, report: observed.report } }
+ : prior;
+ }
+ }
+ // Without a verified snapshot these IDs can only stop a retry. Preserve
+ // the caller's order when they rearranged the same known occurrences.
+ let sourceFindingIds = selection.previous.sourceFindingIds;
+ if (current.findings.length === sourceFindingIds.length) {
+ const members = new Set(sourceFindingIds);
+ const ordered = current.findings.map((row) => row.source_finding_id);
+ if (ordered.every((id) => members.has(id))) sourceFindingIds = ordered;
+ }
+ return {
+ sourceFindingIds,
+ unprovenIdPrefixes: selection.previous.unprovenIdPrefixes,
+ uncertainFindingIds: selection.previous.uncertainFindingIds,
+ unprovenOccurrenceCount: selection.previous.unprovenOccurrenceCount,
+ };
+}
diff --git a/sdk/typescript/src/record.ts b/sdk/typescript/src/record.ts
index d29deeb1f..6456e2c6d 100644
--- a/sdk/typescript/src/record.ts
+++ b/sdk/typescript/src/record.ts
@@ -1,3 +1,60 @@
+import { isBoxedPrimitive, isDate, isProxy } from "node:util/types";
+
+/** Native capture reads enumerable own accessors and may run proxy traps. */
+export function canSnapshotWithoutGetters(
+ input: unknown,
+ observe?: (value: object) => boolean | void,
+): boolean {
+ const seen = new Set<object>();
+ const pending: unknown[] = [input];
+ while (pending.length) {
+ const value = pending.pop();
+ if (!value || typeof value !== "object" || seen.has(value)) continue;
+ seen.add(value);
+ if (isProxy(value)) return false;
+ if (observe?.(value) === false) return false;
+ if (isBoxedPrimitive(value) || ArrayBuffer.isView(value) || isDate(value))
+ continue;
+ const prototype: unknown = Object.getPrototypeOf(value);
+ if (
+ !Array.isArray(value) &&
+ prototype !== Object.prototype &&
+ prototype !== null
+ ) {
+ if (
+ !prototype ||
+ typeof prototype !== "object" ||
+ isProxy(prototype) ||
+ Object.getPrototypeOf(prototype) !== Object.prototype ||
+ !Object.getOwnPropertyNames(prototype).every(
+ (key) => key === "constructor" || key === "toJSON",
+ )
+ )
+ return false;
+ }
+ for (const key of Object.keys(value)) {
+ const field = Object.getOwnPropertyDescriptor(value, key)!;
+ if (!Object.hasOwn(field, "value")) return false;
+ if (field.value !== null && typeof field.value === "object")
+ pending.push(field.value);
+ }
+ }
+ return true;
+}
+
+/** Capture historical data only when cloning cannot run a caller's getter. */
+export function cloneWithoutGetters<T>(
+ value: T,
+ observe?: (value: object) => boolean | void,
+): T | undefined {
+ if (!canSnapshotWithoutGetters(value, observe)) return;
+ try {
+ return structuredClone(value);
+ } catch {
+ return;
+ }
+}
+
export function isRecord(value: unknown): value is Record<string, unknown> {
return typeof value === "object" && value !== null && !Array.isArray(value);
}
@@ -9,9 +66,10 @@ export function asRecord(value: unknown): Record<string, unknown> | undefined {
export function canonicalJson(
value: unknown,
compareKeys?: (left: string, right: string) => number,
+ preserveScalar?: (value: object) => boolean,
): string {
return JSON.stringify(value, (_key, child: unknown) => {
- if (isRecord(child)) {
+ if (isRecord(child) && !preserveScalar?.(child)) {
return Object.fromEntries(
Object.keys(child)
.sort(compareKeys)
diff --git a/sdk/typescript/src/runtime.ts b/sdk/typescript/src/runtime.ts
index b77ac396d..436bb1001 100644
--- a/sdk/typescript/src/runtime.ts
+++ b/sdk/typescript/src/runtime.ts
@@ -2191,8 +2191,7 @@ export async function validatePreparedOutputDir(
const canonical = await realpath(path);
requireModelSafeOutputDir(canonical);
validateLocation?.(canonical);
- const entries = await readdir(canonical);
- if (!allowNonempty && entries.length !== 0) {
+ if (!allowNonempty && (await readdir(canonical)).length !== 0) {
throw new OutputDirectoryError(
`Scan output directory must be empty: ${path}`,
);
diff --git a/sdk/typescript/src/scanner-import.ts b/sdk/typescript/src/scanner-import.ts
index f56a1e68c..3e6dc7d3e 100644
--- a/sdk/typescript/src/scanner-import.ts
+++ b/sdk/typescript/src/scanner-import.ts
@@ -6,6 +6,8 @@ import { gunzip } from "node:zlib";
import { CodexSecurityError, errorMessage } from "./errors.js";
import type { ExternalFinding, ExternalFindings } from "./external-findings.js";
import { isRecord } from "./record.js";
+import { previousAdmission } from "./external-import-contract.js";
+import { rememberPreviousSelection } from "./reader-selection.js";
export function record(value: unknown, label: string): Record<string, unknown> {
if (!value || typeof value !== "object" || Array.isArray(value))
@@ -284,18 +286,130 @@ export function validateSarifRuleReference(
);
}
+interface SizeChange {
+ index: number;
+ position: number;
+ id: string;
+ previous: ExternalFinding["evidence"] | null;
+}
+
+type SelectionFilter = (
+ selection: ExternalFindings,
+ removedIds?: Set<string>,
+) => void;
+
+/** Track older size decisions while a single reader constructs its result. */
+export function trackReaderSelection(result: ExternalFindings) {
+ const changes: SizeChange[] = [];
+ const uncertainIds = new Set<string>();
+
+ function append(
+ position: number,
+ id: string,
+ project: () => ExternalFinding["evidence"],
+ ) {
+ const index = result.findings.length;
+ let outcome: unknown;
+ try {
+ const evidence = project();
+ outcome = evidence;
+ result.findings.push({ source_finding_id: id, evidence });
+ } catch (error) {
+ outcome = error;
+ result.excluded.push({
+ position,
+ source_finding_id: id,
+ reason: errorMessage(error),
+ });
+ }
+ const previous =
+ outcome && typeof outcome === "object"
+ ? previousAdmission(outcome)
+ : undefined;
+ if (previous === "uncertain") uncertainIds.add(id);
+ else if (previous !== undefined)
+ changes.push({ index, position, id, previous });
+ }
+
+ function finish(filter?: SelectionFilter): ExternalFindings {
+ if (!changes.length && !uncertainIds.size) {
+ filter?.(result);
+ return result;
+ }
+ const findings: ExternalFinding[] = [];
+ const excluded: ExternalFindings["excluded"] = [];
+ const restoredPositions = new Set<number>();
+ let cursor = 0;
+ for (const { index, position, id, previous } of changes) {
+ while (cursor < index) findings.push(result.findings[cursor++]!);
+ if (previous) {
+ findings.push({ source_finding_id: id, evidence: previous });
+ restoredPositions.add(position);
+ } else {
+ cursor++;
+ excluded.push({
+ position,
+ source_finding_id: id,
+ reason: "Evidence exceeds the Cloud limit of 256 KiB.",
+ });
+ }
+ }
+ while (cursor < result.findings.length)
+ findings.push(result.findings[cursor++]!);
+ for (const row of result.excluded) {
+ if (!restoredPositions.has(row.position)) excluded.push(row);
+ }
+ excluded.sort((a, b) => a.position - b.position);
+ const previous = { ...result, findings, excluded };
+ // A removed sibling could have depended on an unproven row for its scope.
+ filter?.(previous, uncertainIds.size ? uncertainIds : undefined);
+ filter?.(result);
+ const sourceFindingIds = [
+ ...new Set([
+ ...previous.findings.map((row) => row.source_finding_id),
+ ...uncertainIds,
+ ]),
+ ];
+ if (uncertainIds.size) {
+ rememberPreviousSelection(result, {
+ sourceFindingIds,
+ uncertainFindingIds: [...uncertainIds],
+ knownFindings: structuredClone(
+ previous.findings.filter(
+ (row) => !uncertainIds.has(row.source_finding_id),
+ ),
+ ),
+ });
+ } else {
+ rememberPreviousSelection(result, {
+ sourceFindingIds,
+ snapshot: structuredClone(previous),
+ });
+ }
+ return result;
+ }
+
+ return { append, finish, uncertainIds };
+}
+
/** Exclude every colliding record: choosing the first could overwrite another occurrence. */
export function normalizeRecords<T>(
records: T[],
projectEvidence: (record: T, id: string) => ExternalFinding["evidence"],
identify: (record: T) => string,
target: "local" | "cloud" = "cloud",
+ options: {
+ report?: ExternalFindings["report"];
+ filter?: SelectionFilter;
+ } = {},
): ExternalFindings {
const result: ExternalFindings = {
read: records.length,
findings: [],
excluded: [],
+ ...(options.report === undefined ? {} : { report: options.report }),
};
+ const selection = trackReaderSelection(result);
const identified: { position: number; value: T; id: string }[] = [];
const counts = new Map<string, number>();
for (const [index, value] of records.entries()) {
@@ -331,19 +445,8 @@ export function normalizeRecords<T>(
});
continue;
}
- try {
- result.findings.push({
- source_finding_id: id,
- evidence: projectEvidence(value, id),
- });
- } catch (error) {
- result.excluded.push({
- position,
- source_finding_id: id,
- reason: errorMessage(error),
- });
- }
+ selection.append(position, id, () => projectEvidence(value, id));
}
result.excluded.sort((a, b) => a.position - b.position);
- return result;
+ return selection.finish(options.filter);
}
diff --git a/sdk/typescript/src/semgrep-platform-findings.ts b/sdk/typescript/src/semgrep-platform-findings.ts
index bff161c84..49fa8d7ac 100644
--- a/sdk/typescript/src/semgrep-platform-findings.ts
+++ b/sdk/typescript/src/semgrep-platform-findings.ts
@@ -1,4 +1,7 @@
-import { validateExternalEvidence } from "./external-import-contract.js";
+import {
+ canonicalNumber,
+ validateExternalEvidence,
+} from "./external-import-contract.js";
import type { ExternalFindings } from "./external-findings.js";
import { isRecord } from "./record.js";
import { position, weaknessIds } from "./native-json-findings.js";
@@ -50,35 +53,6 @@ function responseRecord(value: unknown, label: string) {
return record(wireJson.isRawJSON(value) ? undefined : value, label);
}
-// Normalize equivalent JSON numbers exactly. Keep very large exponents compact.
-function canonicalNumber(token: string): string {
- const [, sign = "", integer, fraction = "", exponent = "0"] =
- /^(-?)(\d+)(?:\.(\d+))?(?:e([+-]?\d+))?$/iu.exec(token)!;
- const significant = (integer + fraction).replace(/^0+/u, "");
- if (!significant) return "0";
- let end = significant.length;
- while (significant[end - 1] === "0") end--;
- const digits = significant.slice(0, end);
- const point = BigInt(exponent) + BigInt(significant.length - fraction.length);
- // Ordinary evidence follows JSON.stringify’s fixed/scientific notation thresholds.
- if (point > -6n && point <= 21n) {
- const split = Number(point);
- if (split <= 0) return sign + "0." + "0".repeat(-split) + digits;
- if (split >= digits.length) return sign + digits.padEnd(split, "0");
- return sign + digits.slice(0, split) + "." + digits.slice(split);
- }
- const trailing = digits.slice(1);
- const scientific = point - 1n;
- return (
- sign +
- digits.charAt(0) +
- (trailing ? "." + trailing : "") +
- "e" +
- (scientific >= 0n ? "+" : "") +
- scientific
- );
-}
-
function sourceLine(value: unknown): number | null {
if (value == null) return null;
if (!wireJson.isRawJSON(value)) return position(value, "Semgrep source line");
diff --git a/sdk/typescript/src/sonar-findings.ts b/sdk/typescript/src/sonar-findings.ts
index 4fd78679f..87e6368ea 100644
--- a/sdk/typescript/src/sonar-findings.ts
+++ b/sdk/typescript/src/sonar-findings.ts
@@ -349,7 +349,7 @@ export function normalizeSonarFindings(
...scope,
repositoryUrl: repository,
};
- const result = normalizeRecords(
+ return normalizeRecords(
rows,
({ issue, page }, id) => {
const component = requiredText(
@@ -415,8 +415,8 @@ export function normalizeSonarFindings(
},
({ issue }) => requiredText(issue["key"], "Sonar issue key"),
target,
- );
- return { ...result, report: context };
+ { report: context },
+ ) as ExternalFindings & { report: typeof context };
}
export async function readSonarFindings(
diff --git a/sdk/typescript/tests-ts/external-findings-local-limits.test.ts b/sdk/typescript/tests-ts/external-findings-local-limits.test.ts
index 7a3921a1d..17cfbedfe 100644
--- a/sdk/typescript/tests-ts/external-findings-local-limits.test.ts
+++ b/sdk/typescript/tests-ts/external-findings-local-limits.test.ts
@@ -1,11 +1,19 @@
-import { expect, mock, test } from "bun:test";
+import { expect, mock, spyOn, test } from "bun:test";
+import { execFileSync } from "node:child_process";
import { writeFile } from "node:fs/promises";
import { join } from "node:path";
+import {
+ snapshotNormalizedFindings,
+ type ExternalFindings,
+} from "../src/external-findings.js";
import { prepareLocalExternalImport } from "../src/external-findings-import.js";
import { listLocalExternalFindings } from "../src/external-findings-local.js";
import { prepareNormalizedExternalPublication } from "../src/external-findings-publish.js";
import { readExternalFindings } from "../src/external-findings-read.js";
-import { validateExternalEvidence } from "../src/external-import-contract.js";
+import {
+ validateExternalEvidence,
+ validateExternalEvidenceJson,
+} from "../src/external-import-contract.js";
import type {
ExternalFindingEvidence,
Provider,
@@ -342,6 +350,161 @@ test.each(["wiz", "endor"] as const)(
},
);
+test.each([1e-7, 0.00001])(
+ "Cloud counts the serialized numeric value %s at the evidence byte limit",
+ (score) => {
+ const limit = 256 * 1024;
+ const padding =
+ limit - validateExternalEvidenceJson({ scores: [score] }, "cloud");
+ for (const value of [
+ new Number(score),
+ { hidden: 0.00001, toJSON: () => score },
+ { toJSON: () => Object.assign(new Number(1), { valueOf: () => score }) },
+ ]) {
+ expect(
+ validateExternalEvidenceJson({ scores: [value] }, "cloud", padding),
+ ).toBe(limit);
+ expect(() =>
+ validateExternalEvidenceJson({ scores: [value] }, "cloud", padding + 1),
+ ).toThrow("256 KiB");
+ }
+ },
+);
+
+test("Cloud measures caller fields at their exported numeric size", () => {
+ const limit = 256 * 1024;
+ const padding =
+ limit - validateExternalEvidenceJson({ score: 1e-7 }, "cloud");
+ const score = Object.assign(new Number(1), { toJSON: () => 1e-7 });
+ class Metrics {
+ score = score;
+ }
+ for (const input of [
+ { score },
+ new Metrics(),
+ {
+ get score() {
+ return score;
+ },
+ },
+ ]) {
+ expect(validateExternalEvidenceJson(input, "cloud", padding)).toBe(limit);
+ expect(() =>
+ validateExternalEvidenceJson(input, "cloud", padding + 1),
+ ).toThrow("256 KiB");
+ }
+});
+
+test("local reports and findings need only fit in separate JSON strings", () => {
+ const padding = "x".repeat(3_000);
+ const observations: string[] = [];
+ const report = {
+ toJSON(key: string) {
+ observations.push(key);
+ return {
+ padding,
+ toJSON() {
+ throw new Error("Report was serialized twice");
+ },
+ };
+ },
+ };
+ const input: ExternalFindings = {
+ report,
+ get findings() {
+ observations.push("findings");
+ return [
+ {
+ ...normalized(),
+ evidence: { ...normalized().evidence, source_data: { padding } },
+ },
+ ];
+ },
+ read: 1,
+ excluded: [],
+ };
+ const stringify = JSON.stringify;
+ // Exercise the string boundary in CI without allocating hundreds of MiB.
+ const bounded = spyOn(JSON, "stringify").mockImplementation((...args) => {
+ const json: string = Reflect.apply(stringify, JSON, args);
+ if (json?.length > 4_000) throw new RangeError("Invalid string length");
+ return json;
+ });
+ try {
+ const snapshot = snapshotNormalizedFindings(input);
+ expect(snapshot.report).toEqual({ padding });
+ expect(snapshot.findings[0]!.evidence.source_data).toEqual({ padding });
+ expect(observations).toEqual(["report", "findings"]);
+ } finally {
+ bounded.mockRestore();
+ }
+});
+
+test("local numeric evidence and GitHub batches fit in a 64 MiB Node heap", () => {
+ const sdk = join(import.meta.dir, "..");
+ function run(entrypoint: string, script: string) {
+ const source = execFileSync(
+ process.execPath,
+ [
+ "build",
+ join(sdk, "src", entrypoint + ".ts"),
+ "--target=node",
+ "--format=cjs",
+ "--packages=external",
+ ],
+ { encoding: "utf8" },
+ );
+ return execFileSync(
+ "node",
+ ["--max-old-space-size=64", "--input-type=commonjs"],
+ {
+ input: `${source}\n{${script}}`,
+ cwd: sdk,
+ encoding: "utf8",
+ },
+ );
+ }
+ const direct = run(
+ "external-import-contract",
+ `
+ const samples = Array.from({length: 400_000}, (_, index) => index % 10);
+ const input = {title: "Samples", severity: "low", source_data: {samples}};
+ const result = module.exports.validateExternalEvidence(input, "local");
+ process.stdout.write(String(result.source_data.samples.length));
+ `,
+ );
+ expect(direct).toBe("400000");
+
+ for (const kind of ["alert", "advisory"] as const) {
+ const alert = kind === "alert";
+ const record = alert ? importedAlert() : importedAdvisory();
+ const reader = alert
+ ? "normalizeGitHubCodeScanningAlerts"
+ : "normalizeGitHubAdvisories";
+ const samples = alert
+ ? "Array(650_000).fill(1)"
+ : "Buffer.alloc(512 * 1024)";
+ const retained = run(
+ "github-findings",
+ `
+ const record = ${JSON.stringify(record)};
+ const samples = ${samples};
+ const score = Object.assign(new Number(7), {toJSON: () => 9});
+ record.${kind}.samples = samples;
+ record.${kind}.score = score;
+ if (Array.isArray(samples)) samples[samples.length - 1] = score;
+ const result = module.exports.${reader}([record], "local");
+ const row = result.findings[0].evidence.source_data.${kind};
+ const values = Array.isArray(row.samples) ? row.samples : row.samples.data;
+ process.stdout.write(JSON.stringify([values.length, values.at(-1), row.score]));
+ `,
+ );
+ expect(JSON.parse(retained)).toEqual(
+ alert ? [650_000, 9, 9] : [524_288, 0, 9],
+ );
+ }
+});
+
test("local evidence validation keeps structure, paths, links and numeric integrity", () => {
const circular: Record<string, unknown> = {};
circular["self"] = circular;
diff --git a/sdk/typescript/tests-ts/external-findings-local.test.ts b/sdk/typescript/tests-ts/external-findings-local.test.ts
index bbe75501d..c86d5ed5d 100644
--- a/sdk/typescript/tests-ts/external-findings-local.test.ts
+++ b/sdk/typescript/tests-ts/external-findings-local.test.ts
@@ -875,6 +875,110 @@ describe("local normalized scanner findings", () => {
expect(existsSync(stateDirectory)).toBe(false);
});
+ test("local previews and storage capture the same numeric source data", async () => {
+ const { repository, environment } = await fixture();
+ const data = input();
+ let reads = 0;
+ const sourceData = {
+ scores: Object.assign([3], {
+ note: Object.assign(new Number(4), {
+ toJSON() {
+ throw new Error("JSON ignores named array properties");
+ },
+ }),
+ }),
+ get section() {
+ const n = ++reads;
+ return [new Number(n), new Number(n)];
+ },
+ value: Object.assign(new Number(9), {
+ toJSON: (key: string) => (key === "value" ? 0 : 9),
+ }),
+ };
+ data.findings[0]!.evidence.source_data = sourceData;
+ data.report = { value: sourceData.value, alias: sourceData.value };
+ const prepared = await prepareNormalizedLocalImport(
+ data,
+ { repository, ...source },
+ { environment },
+ );
+ const expected = { scores: [3], section: [1, 1], value: 0 };
+ expect(reads).toBe(1);
+ const preview = prepared.preview.findings[0]!.evidence.source_data!;
+ expect(JSON.parse(JSON.stringify(preview))).toEqual(expected);
+ expect(prepared.preview.report).toEqual({ value: 0, alias: 9 });
+ expect((await prepared.import()).report).toEqual(prepared.preview.report);
+ const rows = await listLocalExternalFindings(
+ { repository },
+ { environment },
+ );
+ expect(
+ rows.find((item) => item.sourceFindingId === "scanner-finding-1")
+ ?.evidence.source_data,
+ ).toEqual(expected);
+ });
+
+ test("captures numeric fields before later getters mutate the source", async () => {
+ const { repository, environment } = await fixture();
+ const data = input();
+ class Scores {
+ score = new Number(3);
+ }
+ const scores = new Scores();
+ let reads = 0;
+ Object.defineProperty(scores, "getterScore", {
+ enumerable: true,
+ get() {
+ reads++;
+ scores.score = new Number(7);
+ return Object.assign(new Number(1), { toJSON: () => 5 });
+ },
+ });
+ data.findings[0]!.evidence.source_data = { scores };
+ Object.assign(data.findings[0]!, {
+ vendorCursor: 2 ** 53,
+ previousCursor: new Number(2 ** 53),
+ });
+ const prepared = await prepareNormalizedLocalImport(
+ data,
+ { repository, ...source },
+ { environment },
+ );
+ expect(reads).toBe(1);
+ const expected = { scores: { score: 3, getterScore: 5 } };
+ expect(
+ JSON.parse(
+ JSON.stringify(prepared.preview.findings[0]!.evidence.source_data),
+ ),
+ ).toEqual(expected);
+ await prepared.import();
+ const rows = await listLocalExternalFindings(
+ { repository },
+ { environment },
+ );
+ expect(
+ rows.find((item) => item.sourceFindingId === "scanner-finding-1")
+ ?.evidence.source_data,
+ ).toEqual(expected);
+ });
+
+ test.each([2 ** 53, new Number(2 ** 53)])(
+ "rejects unsafe normalized evidence before writing any local state",
+ async (score) => {
+ const { repository, stateDirectory, environment } = await fixture();
+ const data = input();
+ data.findings[0]!.evidence.source_data = { score };
+ await expect(
+ prepareNormalizedLocalImport(
+ data,
+ { repository, ...source },
+ { environment },
+ ),
+ ).rejects.toThrow("safe integer");
+ expect(existsSync(stateDirectory)).toBe(false);
+ },
+ );
+
test("uses the canonical selected checkout and rejects state inside it", async () => {
const { root, repository, environment } = await fixture();
const link = join(root, "repository-link");
diff --git a/sdk/typescript/tests-ts/external-findings-publish.test.ts b/sdk/typescript/tests-ts/external-findings-publish.test.ts
index 5375ffc8b..a4f4a9cff 100644
--- a/sdk/typescript/tests-ts/external-findings-publish.test.ts
+++ b/sdk/typescript/tests-ts/external-findings-publish.test.ts
@@ -1,3 +1,5 @@
+import boxedNumberCheckpoints from "./fixtures/boxed-number-checkpoints.json" with { type: "json" };
+import boxedSonarCheckpoint from "./fixtures/boxed-sonar-checkpoint.json" with { type: "json" };
import {
snykJson,
snykPluginExport,
@@ -22,6 +24,7 @@ import { normalizeEndorFindings } from "../src/endor-findings.js";
import { createServer, type ServerResponse } from "node:http";
import { execFile } from "node:child_process";
import { hash } from "node:crypto";
+import { runInNewContext } from "node:vm";
import * as fs from "node:fs/promises";
import {
mkdtemp,
@@ -44,9 +47,19 @@ import {
prepareNormalizedExternalPublication,
type ExternalPublicationOptions,
type ExternalPublicationProgress,
+ type ExternalPublicationResult,
+ type NormalizedExternalPublicationOptions,
type PreparedExternalPublication,
} from "../src/external-findings-publish.js";
-import { readGitHubFindings } from "../src/github-findings.js";
+import {
+ snapshotNormalizedFindings,
+ type ExternalFindings,
+} from "../src/external-findings.js";
+import {
+ normalizeGitHubAdvisories,
+ normalizeGitHubCodeScanningAlerts,
+ readGitHubFindings,
+} from "../src/github-findings.js";
import { readVendorFindings } from "../src/wiz-findings.js";
import {
cloudRepositoryUrlKey,
@@ -61,6 +74,7 @@ import type {
} from "../src/external-import-models.js";
import { main } from "../src/cli.js";
import { dependencies } from "./cli-fixtures.js";
+import { normalizeGitHubDependabotAlerts } from "../src/github-dependabot-findings.js";
import { githubDependabotAlert } from "./support/github-dependabot-fixtures.js";
import {
importedAlert,
@@ -100,6 +114,75 @@ const normalized = (id = "vendor-1", severity = "high") => ({
},
});
+test.each(["local", "cloud"] as const)(
+ "checks raw numbers before capturing normalized evidence and reports on %s",
+ (target) => {
+ const raw = (JSON as JSON & { rawJSON(token: string): unknown }).rawJSON;
+ const capture = (metric: unknown, inReport = false) => {
+ const data: ExternalFindings = {
+ read: 1,
+ findings: [
+ { ...normalized(), evidence: { title: "Example", severity: "high" } },
+ ],
+ excluded: [],
+ };
+ if (inReport) data.report = { metric };
+ else data.findings[0]!.evidence.source_data = { metric };
+ return snapshotNormalizedFindings(data, undefined, target);
+ };
+ for (const inReport of [true, false]) {
+ for (const token of ["1.0000000000000001", "-1e-400"])
+ expect(() => capture(raw(token), inReport)).toThrow("loses precision");
+ expect(() => capture(raw("1e400"), inReport)).toThrow("finite JSON");
+ expect(() => capture(raw("9007199254740993"), inReport)).toThrow(
+ "safe integer",
+ );
+ }
+ expect(capture(raw("12.50E-1")).findings[0]?.evidence.source_data).toEqual({
+ metric: 1.25,
+ });
+ expect(capture(1e-7, true).report).toEqual({ metric: 1e-7 });
+ expect(capture(raw('"1.0000000000000001"'), true).report).toEqual({
+ metric: "1.0000000000000001",
+ });
+ },
+);
+
+test("Cloud stops before expanding a shared subtree beyond its JSON depth limit", async () => {
+ const f = await fixture();
+ let reads = 0;
+ let subtree: Record<string, unknown> = {
+ get score() {
+ reads++;
+ return 7;
+ },
+ };
+ for (let i = 0; i < 7; i++) subtree = { left: subtree, right: subtree };
+ for (let i = 0; i < 20; i++) subtree = { child: subtree };
+ const input: ExternalFindings = {
+ read: 1,
+ findings: [
+ {
+ source_finding_id: "shared-tree",
+ evidence: {
+ title: "Shared tree",
+ severity: "high",
+ source_data: subtree,
+ },
+ },
+ ],
+ excluded: [],
+ };
+
+ await expect(
+ prepareNormalizedExternalPublication(input, options, f.deps),
+ ).rejects.toThrow("Cloud JSON depth limit of 20");
+ expect(reads).toBe(0);
+ expect(f.posts).toHaveLength(0);
+ expect(snapshotNormalizedFindings(input).findings).toHaveLength(1);
+ expect(reads).toBe(128);
+});
+
test("external publication requires opt-in before reading input or accessing Cloud", async () => {
const root = await mkdtemp(join(tmpdir(), "external-import-disabled-"));
cleanups.push(() => rm(root, { recursive: true, force: true }));
@@ -304,6 +387,97 @@ test("a Wiz Raw Event export explains the finding-report path before contacting
expect(contactedCloud).toBe(false);
});
+test("Dependabot keeps an older approval when an emitted boxed number excludes a row", async () => {
+ const f = await fixture();
+ const selection = {
+ ...options,
+ provider: "github" as const,
+ sourceKey: "github.com/example/project/dependabot",
+ };
+ const a = githubDependabotAlert(1);
+ const b = { ...githubDependabotAlert(2), score: new Number(NaN) };
+ const prepare = (rows: unknown[], target: "local" | "cloud" = "cloud") =>
+ prepareReleased(
+ f,
+ normalizeGitHubDependabotAlerts(rows, target),
+ selection,
+ );
+ expect((await prepare([a, b])).preview.requests[0]!.items).toHaveLength(1);
+
+ // The released reader admitted both rows and serialized the non-finite wrapper to null.
+ const request = {
+ ...(await prepare([a, { ...b, score: null }])).preview.requests[0]!,
+ request_id: "e1a703b7-7e69-4a6b-a6c5-02ff7a47fd45",
+ };
+ expect(hash("sha256", JSON.stringify(request))).toBe(
+ "c26b480acaefaa832105d472abf05a372c9eb3d7c0749c15700e802466cb4b3a",
+ );
+ const { path, saved } = await stageReleasedCheckpoint(f, {
+ filename:
+ "a79e3af4845c4c7b5e0abffb5ed37054bb2debdf57b8a8b10acf3d7bd36a5e9f.pending.json",
+ requests: [request],
+ inputDigest:
+ "7b23cc522ea792cfa321415c62c5732421fb63f5c1efbba9fa6f81fb086c9e7f",
+ });
+
+ for (const target of ["local", "cloud"] as const) {
+ await expect(prepare([a, b], target)).rejects.toThrow(
+ "checkpoint was preserved",
+ );
+ }
+ expect(f.posts).toHaveLength(0);
+ expect(await readFile(path, "utf8")).toBe(saved);
+ const changed = {
+ ...a,
+ security_advisory: { ...a.security_advisory, summary: "Changed advisory" },
+ };
+ expect((await prepare([changed, b])).preview.resumed).toBe(false);
+});
+
+test("Dependabot retries the approved subset when a numeric size correction admits another alert", async () => {
+ const f = await fixture();
+ const selection = {
+ ...options,
+ provider: "github" as const,
+ sourceKey: "github.com/example/project/dependabot",
+ };
+ const a = githubDependabotAlert(1);
+ const b = {
+ ...githubDependabotAlert(2),
+ score: new Number(0.00001),
+ pad: "",
+ };
+ const bare = normalizeGitHubDependabotAlerts([b], "cloud").findings[0]!
+ .evidence;
+ b.pad = "x".repeat(256 * 1024 + 1 - Buffer.byteLength(JSON.stringify(bare)));
+ const normalized = normalizeGitHubDependabotAlerts([a, b], "cloud");
+ const prepare = () => prepareReleased(f, normalized, selection);
+ const fresh = (await prepare()).preview.requests[0]!;
+ expect(fresh.items).toHaveLength(2);
+
+ // Recorded from the released reader when only alert 1 fit its size calculation.
+ const request = {
+ ...fresh,
+ request_id: "95af968c-8f7c-4cb5-a79a-561eb06e39e8",
+ items: fresh.items.slice(0, 1),
+ };
+ expect(hash("sha256", JSON.stringify(request))).toBe(
+ "bdec6e68ba764e63a60c83e14d7b691b03332c7af1c4ac031c2b5ca1cb92c69f",
+ );
+ await stageReleasedCheckpoint(f, {
+ filename:
+ "c99d3e8e89437eeb0462959cc3504183d1c67b8bc5a07bfe0af58da058bca72d.pending.json",
+ requests: [request],
+ inputDigest:
+ "50c57ce23ee63e16bd06580d955f37110cdc827b26b4737c58ae13bfb88c2bca",
+ });
+
+ const resumed = await prepare();
+ expect(resumed.preview.resumed).toBe(true);
+ expect((await resumed.publish()).receipts[0]!.id).toBe(request.request_id);
+ expect(f.posts).toEqual([JSON.stringify(request)]);
+});
+
test("Dependabot publishes on the existing GitHub provider and rejects foreign repositories before POST", async () => {
const input = { ...githubDependabotAlert(), source: "extra" };
const correct = await fixture([input]);
@@ -859,17 +1033,18 @@ test("normalized Sonar publication checks retained bindings on preview and retry
},
};
const selection = { read: 2, findings: [finding, named], excluded: [] };
- const prepare = () =>
- prepareNormalizedExternalPublication(
- { ...selection, report: { ...context, branch: "main" } },
+ const prepare = (findings = selection.findings) =>
+ prepareReleased(
+ f,
+ { ...selection, findings, report: { ...context, branch: "main" } },
selected,
- f.deps,
);
+ await expect(prepareReleased(f, selection, selected)).rejects.toThrow(
+ "without a ref",
+ );
await expect(
- prepareNormalizedExternalPublication(selection, selected, f.deps),
- ).rejects.toThrow("without a ref");
- await expect(
- prepareNormalizedExternalPublication(
+ prepareReleased(
+ f,
{
...rows,
report: { ...context, branch: "main" },
@@ -881,14 +1056,13 @@ test("normalized Sonar publication checks retained bindings on preview and retry
],
},
selected,
- f.deps,
),
).rejects.toThrow("retained Sonar branch");
await expect(
- prepareNormalizedExternalPublication(
+ prepareReleased(
+ f,
{ ...rows, report: { ...context, repositoryUrl: foreign } },
selected,
- f.deps,
),
).rejects.toThrow("does not match the selected Cloud repository");
f.state.loseResponse = true;
@@ -896,10 +1070,10 @@ test("normalized Sonar publication checks retained bindings on preview and retry
"resume the saved request",
);
await expect(
- prepareNormalizedExternalPublication(
+ prepareReleased(
+ f,
{ ...selection, report: { ...context, branch: "other" } },
selected,
- f.deps,
),
).rejects.toThrow("conflicting branch");
f.state.repositoryUrl = foreign;
@@ -913,6 +1087,36 @@ test("normalized Sonar publication checks retained bindings on preview and retry
await retry.publish();
expect(f.posts).toHaveLength(2);
expect(f.posts[1]).toBe(f.posts[0]);
+
+ // This old boxed update relies on the report for issue-1's branch on every retry.
+ const { filename, checkpoint } = boxedSonarCheckpoint;
+ const savedPath = join(
+ f.root,
+ "state",
+ "external-finding-publications",
+ filename,
+ );
+ await writeFile(savedPath, JSON.stringify(checkpoint), { mode: 0o600 });
+ const withEffort = (effort: unknown) => {
+ const updated = {
+ ...finding,
+ evidence: {
+ ...finding.evidence,
+ source_data: { ...finding.evidence.source_data, effort },
+ },
+ };
+ return prepare([updated, named]);
+ };
+ f.state.throttle = true;
+ const boxed = await withEffort(new Number(2.5));
+ expect(boxed.preview.resumed).toBe(true);
+ await expect(boxed.publish()).rejects.toThrow("Slow down");
+ f.state.throttle = false;
+ const primitive = await withEffort(2.5);
+ expect(primitive.preview.resumed).toBe(true);
+ expect((await primitive.publish()).verified).toBe(2);
+ const approved = JSON.stringify(checkpoint.requests[0]);
+ expect(f.posts.slice(2)).toEqual([approved, approved]);
});
test("Wiz occurrence mapping preserves original evidence without inventing repository provenance", async () => {
@@ -2039,7 +2243,7 @@ test.each([false, true])(
const firstBody = f.posts[0];
f.state.holdReadback = null;
const second = await run().result;
- expect(second.code).toBe(0);
+ expect(second.code, second.stderr).toBe(0);
if (abrupt) expect(f.posts).toHaveLength(1);
else expect(f.posts[1]).toBe(firstBody);
expect(f.receipts.size).toBe(1);
@@ -2716,6 +2920,149 @@ test.each(["request", "counts", "length", "schema"])(
},
);
+test("an opaque retry preserves a released partial submission whose final row exceeds the corrected limit", async () => {
+ const f = await fixture();
+ const rows = Array.from({ length: 101 }, (_, i) => {
+ const row = normalized(`item-${i}`);
+ return { ...row, evidence: { ...row.evidence, severity: "high" as const } };
+ });
+ const pad = "x".repeat(261872);
+ Object.assign(rows[100]!.evidence, {
+ source_data: {
+ get score() {
+ return Object.assign(new Number(1e-7), { toJSON: () => 0 });
+ },
+ pad,
+ },
+ });
+ const prepare = (selected = rows, sourceKey = options.sourceKey) =>
+ prepareReleased(
+ f,
+ { read: selected.length, findings: selected, excluded: [] },
+ { ...options, sourceKey },
+ );
+ const ready = await prepare();
+ const [first, last] = ready.preview.requests;
+ const [acknowledged] = (await (await prepare(rows.slice(0, 100))).publish())
+ .receipts;
+
+ // Rebuild the released SDK's checkpoint, including its first 100 acknowledged items.
+ first!.request_id = "353fb2dc-0e4a-4be5-b825-5755e4678aa5";
+ last!.request_id = "66da2566-e3c5-4557-aef2-10ffd179982d";
+ last!.items[0]!.evidence.source_data = {
+ score: 1e-7,
+ pad,
+ };
+ const digest =
+ "ae9fd2d8c0f71c489d99604663201894d8d2fe91550bb949035177c8bd0595ef";
+ const saved = JSON.stringify({
+ accountId: "synthetic-account",
+ requests: [first, last],
+ receipts: [{ ...acknowledged, id: first!.request_id }],
+ inputProjection: { inputDigest: digest, savedEvidenceDigest: digest },
+ });
+ expect(hash("sha256", saved)).toBe(
+ "4ca49c21887a29fdb56ea1bcb8197ef5fed02c68f7aacfaa0cc06d9c9b5a128a",
+ );
+ const path = join(
+ f.root,
+ "state",
+ "external-finding-publications",
+ "b8d2c1dda3201683c738fb810639279ba8e30a5dfd1402288a5d7017f7b572e8.pending.json",
+ );
+ await writeFile(path, saved, { mode: 0o600 });
+
+ await expect(prepare()).rejects.toThrow("checkpoint was preserved");
+ await expect(ready.publish()).rejects.toThrow("checkpoint was preserved");
+ f.reports.clear(); // The separate source has no prior findings.
+ expect((await prepare(rows, "tenant-example/other")).preview.resumed).toBe(
+ false,
+ );
+ expect(f.posts).toHaveLength(1);
+ expect(await readFile(path, "utf8")).toBe(saved);
+});
+
+test("a held retry reports the acknowledged batch after another resumer retires the checkpoint", async () => {
+ const f = await fixture(
+ Array.from({ length: 101 }, (_, index) => normalized(`item-${index}`)),
+ );
+ const prepare = () => prepareExternalPublication(f.file, options, f.deps);
+ f.state.postBudget = 1;
+ const original = await prepare();
+ await expect(original.publish()).rejects.toThrow("Slow down");
+ const receipt = f.receipts.get(original.preview.requests[0]!.request_id)!;
+ expect(receipt.item_count).toBe(100);
+ const first = await prepare();
+ const held = await prepare();
+ f.state.postBudget = Infinity;
+ f.state.marker = "generation-2";
+ await expect(first.publish()).rejects.toThrow("HTTP 409");
+
+ const error = await held.publish().catch((error: unknown) => error);
+ if (!(error instanceof ExternalPublicationError)) throw error;
+ expect(error.result).toMatchObject({
+ status: "interrupted",
+ counts: { created: 100, updated: 0, unchanged: 0, error: 0 },
+ unacknowledged: 1,
+ });
+ expect(error.result.receipts).toEqual([receipt]);
+ expect(error.result).not.toHaveProperty("savedSubmission");
+ const bodies = original.preview.requests.map((request) =>
+ JSON.stringify(request),
+ );
+ expect(f.posts).toEqual([bodies[0]!, bodies[1]!, bodies[1]!]);
+ const directory = join(f.root, "state", "external-finding-publications");
+ expect(
+ (await readdir(directory)).filter((name) => name.endsWith(".pending.json")),
+ ).toEqual([]);
+});
+
+test.each(["matching", "replaced", "missing"])(
+ "a held ordinary retry preserves the newer pending request when its original result is %s",
+ async (terminal) => {
+ const f = await fixture(
+ Array.from({ length: 101 }, (_, index) => normalized(`item-${index}`)),
+ );
+ const prepare = () => prepareExternalPublication(f.file, options, f.deps);
+ f.state.postBudget = 1;
+ await expect((await prepare()).publish()).rejects.toThrow("Slow down");
+ const held = await prepare();
+ f.state.postBudget = Infinity;
+ const completed = await (await prepare()).publish();
+ if (terminal === "replaced") await (await prepare()).publish();
+
+ const fresh = await prepare();
+ expect(fresh.preview.requests[0]!.request_id).not.toBe(
+ held.preview.requests[0]!.request_id,
+ );
+ f.state.postBudget = 0;
+ const staged = await fresh.publish().catch((error: unknown) => error);
+ if (!(staged instanceof ExternalPublicationError)) throw staged;
+ expect(staged.message).toContain("HTTP 429");
+ const path = staged.result.savedSubmission!;
+ const saved = await readFile(path, "utf8");
+ if (terminal === "missing")
+ await rm(path.replace(/\.pending\.json$/, ".result.json"));
+
+ const posts = [...f.posts];
+ if (terminal === "matching") {
+ expect(await held.publish()).toEqual(completed);
+ } else {
+ const error = await held.publish().catch((error: unknown) => error);
+ if (!(error instanceof ExternalPublicationError)) throw error;
+ expect(error.result).toMatchObject({
+ status: "interrupted",
+ counts: { created: 100, updated: 0, unchanged: 0, error: 0 },
+ unacknowledged: 1,
+ });
+ expect(error.result.receipts).toEqual([completed.receipts[0]!]);
+ expect(error.result).not.toHaveProperty("savedSubmission");
+ }
+ expect(f.posts).toEqual(posts);
+ expect(await readFile(path, "utf8")).toBe(saved);
+ },
+);
+
test.each([1, 101])(
"a prepared retry keeps its receipts after another resumer completes (%i items)",
async (count) => {
@@ -2731,13 +3078,12 @@ test.each([1, 101])(
const first = await prepareExternalPublication(f.file, options, f.deps);
const second = await prepareExternalPublication(f.file, options, f.deps);
f.state.postBudget = Infinity;
- expect((await first.publish()).counts.created).toBe(count);
+ const completed = await first.publish();
+ expect(completed.counts.created).toBe(count);
const completedPosts = f.posts.length;
- f.state.postBudget = count === 1 ? 0 : 1;
- expect((await second.publish()).counts.created).toBe(count);
- expect(f.posts.slice(completedPosts)).toEqual(
- count === 1 ? [] : [JSON.stringify(second.preview.requests[1])],
- );
+ f.state.postBudget = 0;
+ expect(await second.publish()).toEqual(completed);
+ expect(f.posts).toHaveLength(completedPosts);
const fresh = await prepareExternalPublication(f.file, options, f.deps);
expect(fresh.preview.resumed).toBe(false);
expect(fresh.preview.requests[0]!.request_id).not.toBe(
@@ -4090,69 +4436,1479 @@ test.each(["code-scanning", "advisory"])(
},
);
-test("normalized SDK publication rejects nonfinite raw evidence without JSON coercion", async () => {
- const f = await fixture();
- const input = {
+function scoredFindings(
+ score: unknown,
+ report?: Record<string, unknown>,
+ aliasSelection = false,
+) {
+ const evidence = {
+ title: "Scanner score",
+ severity: "high" as const,
+ source_data: { score },
+ };
+ return {
read: 1,
+ excluded: [],
findings: [
{
- source_finding_id: "nonfinite-raw-evidence",
- evidence: {
- title: "Synthetic evidence",
- severity: "high" as const,
- source_data: { score: Infinity },
- },
+ source_finding_id: "boxed-score",
+ selection: aliasSelection ? evidence : "security", // Hashed by older publishers but absent from requests.
+ evidence,
},
],
- excluded: [],
+ report,
};
- await expect(
- prepareNormalizedExternalPublication(input, options, f.deps),
- ).rejects.toThrow("finite JSON numbers");
- expect(input.findings[0]!.evidence.source_data.score).toBe(Infinity);
- expect(f.posts).toHaveLength(0);
-});
+}
-test("normalized SDK publication rejects unserializable report metadata before upload", async () => {
+type PublicationFixture = Awaited<ReturnType<typeof fixture>>;
+
+// Historical filenames include the deployment. Use the one recorded in these fixtures.
+function prepareReleased(
+ f: PublicationFixture,
+ input: ExternalFindings,
+ selected: NormalizedExternalPublicationOptions = options,
+) {
+ return prepareNormalizedExternalPublication(input, selected, {
+ ...f.deps,
+ environment: {
+ ...f.environment,
+ CODEX_SECURITY_CLOUD_BASE_URL: undefined,
+ CODEX_SECURITY_CLOUD_WEB_URL: undefined,
+ },
+ });
+}
+
+async function stageReleasedCheckpoint(
+ f: PublicationFixture,
+ {
+ filename,
+ requests,
+ inputDigest,
+ savedEvidenceDigest = inputDigest,
+ accountId = boxedNumberCheckpoints.accountId,
+ }: {
+ filename: string;
+ requests: readonly unknown[];
+ inputDigest: string;
+ savedEvidenceDigest?: string;
+ accountId?: string;
+ },
+) {
+ const directory = join(f.root, "state", "external-finding-publications");
+ const path = join(directory, filename);
+ const checkpoint = {
+ accountId,
+ requests,
+ inputProjection: { inputDigest, savedEvidenceDigest },
+ };
+ const saved = JSON.stringify(checkpoint);
+ await mkdir(directory, { recursive: true, mode: 0o700 });
+ await writeFile(path, saved, { mode: 0o600 });
+ return { checkpoint, directory, path, saved };
+}
+
+// These filenames, request IDs and projections were recorded by the old SDK.
+async function oldScoreApproval(field: string, stageApproval = true) {
const f = await fixture();
- const input = {
- read: 1,
- report: { elapsedNs: 1n },
- findings: [
- {
- source_finding_id: "synthetic-report",
- evidence: { title: "Example", severity: "unknown" as const },
+ const { filename, requestId, inputProjection } =
+ boxedNumberCheckpoints.cases.find((item) => item.field === field)!;
+ const prepare = (
+ score: unknown,
+ report?: Record<string, unknown>,
+ flag?: unknown,
+ ) => {
+ const input = scoredFindings(score, report, field === "aliased");
+ if (flag !== undefined)
+ Object.assign(input.findings[0]!.evidence.source_data, { flag });
+ return prepareReleased(f, input);
+ };
+ const request = {
+ request_id: requestId,
+ ...structuredClone(boxedNumberCheckpoints.request),
+ };
+ if (field === "flag") {
+ request.items[0]!.expected_version = 0;
+ Object.assign(request.items[0]!.evidence.source_data, {
+ score: 12,
+ flag: {},
+ });
+ } else {
+ if (field === "empty-report")
+ request.items[0]!.evidence.source_data.score = 12;
+ await (await prepare(0)).publish(); // The old checkpoint approved an update.
+ f.posts.length = 0;
+ }
+ const directory = join(f.root, "state", "external-finding-publications");
+ const path = join(directory, filename);
+ const saved = JSON.stringify({
+ accountId: boxedNumberCheckpoints.accountId,
+ requests: [request],
+ inputProjection,
+ });
+ if (stageApproval) {
+ await mkdir(directory, { recursive: true, mode: 0o700 });
+ await writeFile(path, saved, { mode: 0o600 });
+ }
+ return { f, prepare, directory, path, filename, request, saved };
+}
+
+test.each(["invalid title", "unsafe number", "cross-realm serializer error"])(
+ "a fresh %s can only publish its verified older request",
+ async (kind) => {
+ const { f, path, saved, request } = await oldScoreApproval(
+ "evidence",
+ false,
+ );
+ const score = new Number(2.5);
+ const selected = scoredFindings(score);
+ if (kind === "cross-realm serializer error") {
+ Object.defineProperty(score, "toJSON", {
+ value: runInNewContext(
+ '() => { throw new TypeError("invalid metric") }',
+ ),
+ });
+ } else if (kind === "unsafe number") {
+ Object.defineProperty(score, "toJSON", { value: () => 2 ** 53 });
+ } else {
+ Object.setPrototypeOf(selected.findings[0]!.evidence, {
+ toJSON(this: Record<string, unknown>) {
+ return { ...this, title: "" };
+ },
+ });
+ }
+ const prepare = () => prepareReleased(f, selected);
+ await expect(prepare()).rejects.toThrow();
+ expect(f.posts).toHaveLength(0);
+ await writeFile(path, saved, { mode: 0o600 });
+ const recovered = await prepare();
+ expect(recovered.preview.resumed).toBe(true);
+ await recovered.publish();
+ expect(f.posts).toEqual([JSON.stringify(request)]);
+ },
+);
+
+test.each(["deep shared data", "cycle", "row BigInt"])(
+ "a compact export remains publishable with an unusable native %s",
+ async (kind) => {
+ const f = await fixture();
+ const input = scoredFindings(4);
+ if (kind === "row BigInt") {
+ Object.assign(input.findings[0]!, {
+ cursor: Object.assign(Object(17n), { toJSON: () => "17" }),
+ });
+ } else {
+ class Compact {
+ constructor(readonly content: unknown) {}
+ toJSON() {
+ return 4;
+ }
+ }
+ let content: unknown = {};
+ if (kind === "cycle") Object.assign(content as object, { self: content });
+ else for (let n = 0; n < 24; n++) content = [content, content];
+ input.findings[0]!.evidence.source_data.score = new Compact(content);
+ }
+ const preview = await prepareReleased(f, input);
+ expect(preview.preview.resumed).toBe(false);
+ await preview.publish();
+ expect(JSON.parse(f.posts[0]!).items[0].evidence.source_data.score).toBe(4);
+ },
+);
+
+test.each(["getter", "data property"])(
+ "a JSON hook cannot replace a %s score to match a released approval",
+ async (kind) => {
+ const { f, path, saved } = await oldScoreApproval("evidence");
+ const input = scoredFindings(undefined);
+ const data = input.findings[0]!.evidence.source_data;
+ const score = Object.assign(new Number(8), {
+ toJSON() {
+ Object.defineProperty(data, "score", { value: new Number(2.5) });
+ return 9;
},
- ],
- excluded: [],
+ });
+ if (kind === "getter")
+ Object.defineProperty(data, "score", {
+ configurable: true,
+ enumerable: true,
+ get: () => score,
+ });
+ else data.score = score;
+
+ await expect(prepareReleased(f, input)).rejects.toThrow();
+ expect(await readFile(path, "utf8")).toBe(saved);
+ expect(f.posts).toHaveLength(0);
+ },
+);
+
+test("a shared mutation across two GitHub alerts preserves their released approval", async () => {
+ const f = await fixture();
+ const prepare = () => {
+ const counter = { n: new Number(0) };
+ const first = {
+ get sample() {
+ counter.n = new Number(Number(counter.n) + 1);
+ return counter.n;
+ },
+ counter,
+ };
+ class Metrics {
+ score = Object.assign(new Number(7), { toJSON: () => 9 });
+ counter = counter;
+ }
+ const rows = [first, new Metrics()].map((extension, i) => {
+ const row = importedAlert();
+ row.number = row.alert.number = 12 + i;
+ row.url = row.alert.html_url = row.url.replace(/12$/, String(row.number));
+ Object.assign(row.alert, { extension });
+ return row;
+ });
+ return prepareReleased(f, normalizeGitHubCodeScanningAlerts(rows), {
+ ...options,
+ provider: "github",
+ });
};
- await expect(
- prepareNormalizedExternalPublication(input, options, f.deps),
- ).rejects.toThrow(/bigint/i);
- await expect(
- prepareNormalizedExternalPublication(
- { ...input, report: { elapsed: new Number(Infinity) } },
- options,
- f.deps,
- ),
- ).rejects.toThrow("finite JSON numbers");
+
+ const request = JSON.parse(
+ JSON.stringify((await prepare()).preview.requests[0]),
+ );
+ const first = request.items[0].evidence.source_data.alert.extension;
+ const second = request.items[1].evidence.source_data.alert.extension;
+ expect([first.counter.n, second.counter.n, second.score]).toEqual([1, 1, 9]);
+ // The released publisher captured both alerts after three getter reads.
+ first.sample = first.counter.n = second.counter.n = 3;
+ second.score = 7;
+ request.request_id = "035de7b2-7e71-4166-be9a-99ae1251217b";
+ expect(hash("sha256", JSON.stringify(request))).toBe(
+ "8ce6e5a6d4be2062a39ba7325554f7849096b472a67ae1672fd614c681db9a37",
+ );
+ const { path, saved } = await stageReleasedCheckpoint(f, {
+ filename:
+ "a7b6eecd18536c47e5bcd5e1db25a45b3faca1715eab39e2884b06bf67b1e62f.pending.json",
+ requests: [request],
+ inputDigest:
+ "1fa6cbfd77fccdeec029930585f46595118e9f95b8083480239ac50176fb9a52",
+ });
+
+ await expect(prepare()).rejects.toThrow("cannot be matched");
expect(f.posts).toHaveLength(0);
+ expect(await readFile(path, "utf8")).toBe(saved);
});
-test("publication JSON retains native Error messages and scanner diagnostics", async () => {
- const scannerErrors = opengrepJson().errors;
- const error = new Error("Scanner rule timed out.");
- const nested = {
- path: "src/example.ts",
- cause: new Error("Parser stopped."),
+test("a Sonar report resumes its released request after a size change", async () => {
+ const { normalizeSonarFindings: normalize } =
+ await import("../src/sonar-findings.js");
+ const repository = "https://github.com/example/project";
+ const issue = {
+ key: "issue-one",
+ project: "example",
+ component: "example:one",
+ rule: "example:S1",
+ message: "Synthetic finding",
+ type: "VULNERABILITY",
};
- const report = { errors: [error, nested, ...scannerErrors] };
- const jsonErrors = [
- { message: error.message },
- { path: nested.path, cause: { message: nested.cause.message } },
- ...scannerErrors,
- ];
- const f = await fixture();
+ const boundary = {
+ ...issue,
+ key: "issue-two",
+ score: new Number(0.00001),
+ pad: "",
+ };
+ const raw = {
+ paging: { pageIndex: 1, pageSize: 10, total: 2 },
+ issues: [issue, boundary],
+ };
+ const base = normalize(raw, repository);
+ boundary.pad = "p".repeat(
+ 262_145 - Buffer.byteLength(JSON.stringify(base.findings[1]!.evidence)),
+ );
+ const selection = normalize(raw, repository);
+ const f = await fixture();
+ const selected = { ...options, provider: "sonar" as const };
+ const prepare = () => prepareReleased(f, selection, selected);
+ const fresh = await prepare();
+ expect(fresh.preview.findings.map((row) => row.source_finding_id)).toEqual([
+ "issue-one",
+ "issue-two",
+ ]);
+ // The earlier SDK approved only the first issue and retained the Sonar report.
+ const batch = fresh.preview.requests[0]!;
+ const request = {
+ ...batch,
+ request_id: "bd9002e4-5abe-469b-ad2d-a64514f82e1b",
+ items: [batch.items[0]!],
+ };
+ expect(hash("sha256", JSON.stringify(request))).toBe(
+ "e6bc41119d363155e2d2388985ce3b476dfd3c6898297b5154a9b09ab192ac95",
+ );
+ await stageReleasedCheckpoint(f, {
+ filename:
+ "bb2529dff62d8c9bb1c064d18f2b649c412b3d310914a8d4221c115a00f71e4c.pending.json",
+ requests: [request],
+ inputDigest:
+ "62f0d447578728d3c8afa0ebfa0d1a2d8d7ab5c9d1a96af2855508bc5e6da986",
+ savedEvidenceDigest:
+ "25716f8958e0b38376fe5fdaad85b648ca3b0ed23a86c37a246a1537a75c848d",
+ });
+ const retry = await prepare();
+ expect(retry.preview.resumed).toBe(true);
+ expect(retry.preview.report).toEqual(selection.report);
+ expect(retry.preview.findings.map((row) => row.source_finding_id)).toEqual([
+ "issue-one",
+ ]);
+ expect((await retry.publish()).verified).toBe(1);
+ expect(f.posts).toEqual([JSON.stringify(request)]);
+});
+
+test("an uncertain newly eligible advisory cannot replace a released subset", async () => {
+ const f = await fixture();
+ const selected = { ...options, provider: "github" as const };
+ const prepare = (rows: ReturnType<typeof importedAdvisory>[]) =>
+ prepareReleased(f, normalizeGitHubAdvisories(rows), selected);
+ const first = importedAdvisory();
+ const id = "GHSA-abcd-efgh-5678";
+ const url = first.url.replace(first.ghsaId, id);
+ const second = {
+ ...first,
+ ghsaId: id,
+ url,
+ advisory: { ...first.advisory, ghsa_id: id, html_url: url },
+ };
+ const value = Object.assign(new Number(7), { bad: Infinity });
+ Object.defineProperty(second.advisory, "score", {
+ enumerable: true,
+ get: () => value,
+ });
+ expect(normalizeGitHubAdvisories([first, second]).findings).toHaveLength(2);
+
+ const request = {
+ ...(await prepare([first])).preview.requests[0]!,
+ request_id: "58b468b5-45c6-4804-8f5a-5e2919c967c5",
+ };
+ expect(hash("sha256", JSON.stringify(request))).toBe(
+ "19039224ccc25a9375739c7f16227a26e9fd92cf900de1a32425c20c94697488",
+ );
+ const { path, saved } = await stageReleasedCheckpoint(f, {
+ filename:
+ "6137d904632ecb17ecdf0276e37794699ff19f6e37f6a89b833b56ec0a7c953b.pending.json",
+ requests: [request],
+ inputDigest:
+ "fbba527f5ff2cf86972d97f543de935515cbaa618ab2f1168ddbb3d1ab3ea338",
+ });
+
+ await expect(prepare([first, second])).rejects.toThrow("cannot be matched");
+ expect(await readFile(path, "utf8")).toBe(saved);
+ expect(f.posts).toEqual([]);
+ expect((await (await prepare([first])).publish()).receipts[0]!.id).toBe(
+ request.request_id,
+ );
+ expect(f.posts).toEqual([JSON.stringify(request)]);
+});
+
+test("an ambiguous getter cannot start another import after a released approval was matched", async () => {
+ const { f, prepare, path, request } = await oldScoreApproval("evidence");
+ const score = Object.assign(new Number(2.5), { toJSON: () => 9 });
+ f.state.throttle = true;
+ await expect((await prepare(score)).publish()).rejects.toThrow("Slow down");
+ const pending = await readFile(path, "utf8");
+ f.state.throttle = false;
+
+ const input = scoredFindings(undefined);
+ Object.defineProperty(input.findings[0]!.evidence.source_data, "score", {
+ enumerable: true,
+ get: () => score,
+ });
+ const publish = async () => (await prepareReleased(f, input)).publish();
+ await expect(publish()).rejects.toThrow();
+ expect(await readFile(path, "utf8")).toBe(pending);
+ expect(f.posts).toEqual([JSON.stringify(request)]);
+
+ const retry = await prepare(score);
+ expect(retry.preview.resumed).toBe(true);
+ expect((await retry.publish()).receipts[0]!.id).toBe(request.request_id);
+ expect(f.posts).toEqual([JSON.stringify(request), JSON.stringify(request)]);
+});
+
+test("a fresh boxed approval cannot claim a different getter-backed JSON import", async () => {
+ const { f, prepare } = await oldScoreApproval("evidence", false);
+ f.state.throttle = true;
+ const first = await prepare(new Number(1));
+ await expect(first.publish()).rejects.toThrow("Slow down");
+ const input = scoredFindings(undefined);
+ Object.defineProperty(input.findings[0]!.evidence.source_data, "score", {
+ enumerable: true,
+ get: () => 2,
+ });
+ const second = await prepareReleased(f, input);
+ expect(second.preview.resumed).toBe(false);
+ expect(second.preview.requests[0]!.request_id).not.toBe(
+ first.preview.requests[0]!.request_id,
+ );
+ expect(second.preview.requests[0]!.items[0]!.evidence.source_data).toEqual({
+ score: 2,
+ });
+ expect(f.posts).toEqual([JSON.stringify(first.preview.requests[0])]);
+});
+
+test("a root serializer cannot add a report to distinguish a released approval", async () => {
+ const { f, request } = await oldScoreApproval("unreported", false);
+ const { source_finding_id, evidence } = scoredFindings(2.5).findings[0]!;
+ class ScannerBatch {
+ read = 1;
+ excluded = [];
+ findings = [{ source_finding_id, evidence }];
+ reportForJson() {
+ return { version: "1" };
+ }
+ toJSON() {
+ return { ...this, report: this.reportForJson() };
+ }
+ }
+ const input = new ScannerBatch();
+ const prepare = () => prepareReleased(f, input);
+ const held = await prepare();
+ const { path, saved } = await stageReleasedCheckpoint(f, {
+ filename:
+ "29fc89c807be65a8b3a173feeb64e86595e3debbdb91dc00b1a9cfb6eea869cc.pending.json",
+ requests: [
+ { ...request, request_id: "7e77248b-2906-44a7-b467-5225966ac7a2" },
+ ],
+ inputDigest:
+ "875a8aec2f1b7c573a6d05d983f6afa0f964a6e1c96d22c8c8691cc2de824ccc",
+ });
+ await expect(prepare()).rejects.toThrow("cannot be matched");
+ await expect(held.publish()).rejects.toThrow("cannot be matched");
+ expect(await readFile(path, "utf8")).toBe(saved);
+ expect(f.posts).toEqual([]);
+ const independent = await prepareReleased(f, {
+ ...input,
+ report: input.reportForJson(),
+ });
+ expect(independent.preview.resumed).toBe(false);
+});
+
+test("a finding serializer cannot replace the ID of an older approval", async () => {
+ const { f, path, saved } = await oldScoreApproval("unreported");
+ const input = scoredFindings(2.5);
+ const stored = input.findings[0]!;
+ const prepare = () => prepareReleased(f, input);
+ class ScannerFinding {
+ exportedId() {
+ return "exported-id";
+ }
+ toJSON() {
+ return { ...stored, source_finding_id: this.exportedId() };
+ }
+ }
+ const finding = Object.assign(new ScannerFinding(), stored);
+ input.findings[0] = finding;
+ await expect(prepare()).rejects.toThrow("cannot be matched");
+ let idReads = 0;
+ Object.defineProperty(finding, "source_finding_id", {
+ enumerable: true,
+ get() {
+ idReads++;
+ return stored.source_finding_id;
+ },
+ });
+ await expect(prepare()).rejects.toThrow("cannot be matched");
+ expect(idReads).toBe(0);
+ expect(await readFile(path, "utf8")).toBe(saved);
+ expect(f.posts).toEqual([]);
+
+ input.findings[0] = { ...stored, source_finding_id: "independent-id" };
+ Object.defineProperty(input.findings[0].evidence.source_data, "score", {
+ enumerable: true,
+ get: () => 2.5,
+ });
+ expect((await prepare()).preview.resumed).toBe(false);
+});
+
+test("a vendor evidence class resumes its released title and exports its JSON title for a fresh import", async () => {
+ class ScannerEvidence {
+ title = "Scanner score";
+ severity = "high" as const;
+ source_data = { score: 2.5 };
+ toJSON() {
+ return { ...this, title: "New scanner score" };
+ }
+ }
+ const { f, path, request, saved } = await oldScoreApproval("unreported");
+ const input = scoredFindings(2.5);
+ input.findings[0]!.evidence = new ScannerEvidence();
+ const prepare = () => prepareReleased(f, input);
+ // Without an independent native capture, the exported title cannot rule out this approval.
+ class UncertainEvidence extends ScannerEvidence {
+ exportedTitle() {
+ return "Separate scanner title";
+ }
+ override toJSON() {
+ return { ...this, title: this.exportedTitle() };
+ }
+ }
+ input.findings[0]!.evidence = new UncertainEvidence();
+ await expect(prepare()).rejects.toThrow("cannot be matched");
+ expect(await readFile(path, "utf8")).toBe(saved);
+ expect(f.posts).toEqual([]);
+
+ input.findings[0]!.evidence = {
+ ...new ScannerEvidence(),
+ title: "Separate scanner title",
+ };
+ expect((await prepare()).preview.resumed).toBe(false);
+ input.findings[0]!.evidence = new ScannerEvidence();
+ const retry = await prepare();
+ expect(retry.preview.resumed).toBe(true);
+ expect((await retry.publish()).receipts[0]!.id).toBe(request.request_id);
+ expect(f.posts).toEqual([JSON.stringify(request)]);
+
+ const fresh = await prepare();
+ expect(fresh.preview.resumed).toBe(false);
+ expect((await fresh.publish()).verified).toBe(1);
+ expect(JSON.parse(f.posts[1]!).items[0].evidence.title).toBe(
+ "New scanner score",
+ );
+});
+
+test("a shared rule in report metadata resumes the released request", async () => {
+ const f = await fixture();
+ const rule = { name: "example-rule" };
+ const input = scoredFindings(new Number(2.5), { rule });
+ Object.assign(input.findings[0]!.evidence, {
+ details: { kind: "sast", repository: { url: f.destination().url }, rule },
+ });
+ const prepare = () => prepareReleased(f, input);
+ const request = {
+ ...(await prepare()).preview.requests[0]!,
+ request_id: "0697e266-e0a5-4aa5-b30b-3b89cbd48ac1",
+ };
+ expect(hash("sha256", JSON.stringify(request))).toBe(
+ "8c76780e8aca7582a293dee9483002401d6907e51833fc806cd1a058fe54b855",
+ );
+ await stageReleasedCheckpoint(f, {
+ filename:
+ "749e46bc07036abe74f047011264147993bd96f384099bcf1b729c8c2634a1b0.pending.json",
+ requests: [request],
+ inputDigest:
+ "49cf7aa8362d8967c5638238b719d925e7e1814d48a4a7a0844e69ef9260b869",
+ savedEvidenceDigest:
+ "a2d3adb20cf97c597828fd54ec361490a8a34e479a42d01d3b85ff245b0b384b",
+ });
+
+ const resumed = await prepare();
+ expect(resumed.preview.resumed).toBe(true);
+ expect((await resumed.publish()).receipts[0]!.id).toBe(request.request_id);
+ expect(f.posts).toEqual([JSON.stringify(request)]);
+});
+
+test("a boxed JSON hook keeps the released request and applies to fresh normalized publication", async () => {
+ const { f, prepare, path, request, saved } =
+ await oldScoreApproval("evidence");
+ const score = (native: number) =>
+ Object.assign(new Number(native), { toJSON: () => 9 });
+
+ await expect(prepare(score(8))).rejects.toThrow();
+ for (const plain of [2.5, JSON.parse(JSON.stringify(score(2.5))), "7.00"])
+ await expect(prepare(plain)).rejects.toThrow();
+ expect(await readFile(path, "utf8")).toBe(saved);
+ expect(f.posts).toHaveLength(0);
+
+ f.state.brokenReadback = true;
+ const migrated = await prepare(new Number(2.5));
+ await expect(migrated.publish()).rejects.toThrow("Readback unavailable");
+ f.state.brokenReadback = false;
+ const resumed = await prepare(score(2.5));
+ expect(resumed.preview.resumed).toBe(true);
+ expect((await resumed.publish()).receipts[0]!.id).toBe(request.request_id);
+ expect(f.posts).toEqual([JSON.stringify(request)]);
+
+ const ordinary = await prepare(new Number(2.5));
+ f.state.brokenReadback = true;
+ await expect(ordinary.publish()).rejects.toThrow("Readback unavailable");
+ const pending = await readFile(path, "utf8");
+ await expect(prepare(score(2.5))).rejects.toThrow("Finish the other pending");
+ expect(await readFile(path, "utf8")).toBe(pending);
+ f.state.brokenReadback = false;
+ expect((await (await prepare(2.5)).publish()).receipts[0]!.id).toBe(
+ ordinary.preview.requests[0]!.request_id,
+ );
+ expect(f.posts).toEqual([
+ JSON.stringify(request),
+ JSON.stringify(ordinary.preview.requests[0]),
+ ]);
+ const fresh = await prepare(score(2.5));
+ expect(fresh.preview.resumed).toBe(false);
+ expect((await fresh.publish()).verified).toBe(1);
+ expect(JSON.parse(f.posts[2]!).items[0].evidence.source_data).toEqual({
+ score: 9,
+ });
+});
+
+test.each([
+ { field: "evidence", native: 2.5, report: undefined },
+ { field: "empty-report", native: 12, report: {} },
+])(
+ "a released $field approval stays on its original deployment",
+ async ({ field, native, report }) => {
+ const { f, path, saved, request } = await oldScoreApproval(field);
+ const input = scoredFindings(
+ Object.assign(new Number(native), { toJSON: () => 9 }),
+ report,
+ );
+ if (report) {
+ class SourceData {
+ score = new Number(native);
+ toJSON() {
+ return { score: 9 };
+ }
+ }
+ input.findings[0]!.evidence.source_data = new SourceData();
+ }
+ const other = await prepareNormalizedExternalPublication(input, options, {
+ ...f.deps,
+ environment: {
+ ...f.environment,
+ CODEX_SECURITY_CLOUD_BASE_URL: "https://other.example.test/backend-api",
+ CODEX_SECURITY_CLOUD_WEB_URL: "https://other.example.test",
+ },
+ });
+ expect(other.preview.resumed).toBe(false);
+ expect(other.preview.requests[0]!.request_id).not.toBe(request.request_id);
+ expect(other.preview.requests[0]!.items[0]!.evidence.source_data).toEqual({
+ score: 9,
+ });
+ expect(f.posts).toHaveLength(0);
+ expect(await readFile(path, "utf8")).toBe(saved);
+ },
+);
+
+test("an opaque retry cannot choose between independent old and new approvals", async () => {
+ const {
+ f,
+ path: oldPath,
+ saved,
+ directory,
+ } = await oldScoreApproval("unreported", false);
+ const exported = scoredFindings(9);
+ exported.findings[0]!.source_finding_id = "exported-row";
+ const opaque = scoredFindings(2.5);
+ const nativeRows = opaque.findings;
+ Object.defineProperty(opaque, "findings", { get: () => nativeRows });
+ Object.defineProperty(opaque, "toJSON", { value: () => exported });
+ const prepare = (input: typeof opaque) => prepareReleased(f, input);
+
+ const newer = await prepare(exported);
+ const request = newer.preview.requests[0]!;
+ const body = JSON.stringify(request);
+ f.state.throttle = true;
+ await expect(newer.publish()).rejects.toThrow("Slow down");
+ const filename = (await readdir(directory)).find((name) =>
+ name.endsWith(".pending.json"),
+ )!;
+ const newPath = join(directory, filename);
+ const newerBytes = await readFile(newPath, "utf8");
+ const held = await prepare(opaque);
+ expect(held.preview.requests[0]!.request_id).toBe(request.request_id);
+ f.state.throttle = false;
+ await writeFile(oldPath, saved, { mode: 0o600 });
+
+ await expect(prepare(opaque)).rejects.toThrow();
+ await expect(held.publish()).rejects.toThrow();
+ expect(f.posts).toEqual([body]);
+ expect(await readFile(oldPath, "utf8")).toBe(saved);
+ expect(await readFile(newPath, "utf8")).toBe(newerBytes);
+ const approved = await prepare(exported);
+ expect((await approved.publish()).receipts[0]!.id).toBe(request.request_id);
+ expect(f.posts).toEqual([body, body]);
+ expect(await readFile(oldPath, "utf8")).toBe(saved);
+});
+
+test.each([false, true])(
+ "a getter-backed approval stays scoped to its deployment (with report: %s)",
+ async (reported) => {
+ const f = await fixture();
+ const finding = {
+ source_finding_id: "synthetic-score",
+ evidence: {
+ title: "Score",
+ severity: "high" as const,
+ source_data: {
+ get score() {
+ return 2.5;
+ },
+ },
+ },
+ };
+ const prepare = (host: string) =>
+ prepareNormalizedExternalPublication(
+ {
+ read: 1,
+ excluded: [],
+ findings: [finding],
+ report: reported ? { version: "1" } : undefined,
+ },
+ options,
+ {
+ ...f.deps,
+ environment: {
+ ...f.environment,
+ CODEX_SECURITY_CLOUD_BASE_URL: `https://${host}.example.test/backend-api/aardvark`,
+ CODEX_SECURITY_CLOUD_WEB_URL: `https://${host}.example.test`,
+ },
+ },
+ );
+ // Recorded with the released SDK on deployment one.
+ const request = {
+ ...(await prepare("one")).preview.requests[0]!,
+ request_id: reported
+ ? "afcc3987-d289-4b85-9e7c-8ff755ebb64b"
+ : "bf6f1063-9647-43d8-82b9-982961522482",
+ };
+ const digest =
+ "01ad76cac2893a22136a81e7e61aeef35edb7a51e97f2944713e744756362721";
+ const { path, saved } = await stageReleasedCheckpoint(f, {
+ filename: reported
+ ? "66fdd41e29ebfe07ca8fe9e54689df0027301c08a19b8b17b82254aadb536b44.pending.json"
+ : "b4eed57e616f2289bf467f18fba85e2e9ba305a360a798d9e9d8125b06e148fe.pending.json",
+ requests: [request],
+ inputDigest: reported
+ ? "af3bca9d5d1b53284f6da3d32b4de04d196cf14f96a0b7a00711dcd6e90e896d"
+ : digest,
+ savedEvidenceDigest: digest,
+ });
+ expect((await prepare("one")).preview.requests).toEqual([request]);
+ const independent = await prepare("two");
+ expect(independent.preview.resumed).toBe(false);
+ expect(independent.preview.requests[0]!.request_id).not.toBe(
+ request.request_id,
+ );
+ expect((await independent.publish()).verified).toBe(1);
+ expect(f.posts).toEqual([JSON.stringify(independent.preview.requests[0])]);
+ expect(await readFile(path, "utf8")).toBe(saved);
+ },
+);
+
+test.each([
+ { first: "reported", reportedScore: 2.5 },
+ { first: "report-free", reportedScore: 2.5 },
+ { first: "reported", reportedScore: 7 },
+])(
+ "distinct released boxed approvals preserve both selections when $first resumes first (reported score $reportedScore)",
+ async ({ first, reportedScore }) => {
+ const { f, prepare, directory, path, request, saved } =
+ await oldScoreApproval("evidence");
+ // The released SDK used this report-bound digest for either boxed score.
+ const reportedPath = join(
+ directory,
+ "ad28704f08feeeaf1adb30d1010204b8ac942fa0c215c5b4eabb4990d338f23a.pending.json",
+ );
+ const recorded = JSON.parse(saved);
+ const reportedRequest = structuredClone(request);
+ reportedRequest.request_id =
+ reportedScore === 7
+ ? "fdfa27bc-13f6-4c83-8abc-fc3619e3b512"
+ : "ec3a5d47-b29e-493a-90ea-effa70148ef4";
+ reportedRequest.items[0]!.evidence.source_data.score = reportedScore;
+ recorded.requests = [reportedRequest];
+ recorded.inputProjection.inputDigest =
+ "53cc56f227da631bd73c1301c623b9d8c55aca65e03feb324808ae9c886ed1da";
+ await writeFile(reportedPath, JSON.stringify(recorded), { mode: 0o600 });
+
+ const reported = {
+ report: { version: "1" },
+ score: reportedScore,
+ path: reportedPath,
+ request: reportedRequest,
+ };
+ const plain = { report: undefined, score: 2.5, path, request };
+ const order = first === "reported" ? [reported, plain] : [plain, reported];
+ f.state.throttle = true;
+ for (const selection of order) {
+ const sibling = selection === reported ? plain : reported;
+ const siblingBytes = await readFile(sibling.path, "utf8");
+ const prepared = await prepare(
+ new Number(selection.score),
+ selection.report,
+ );
+ expect(prepared.preview.resumed).toBe(true);
+ expect(prepared.preview.requests[0]!.request_id).toBe(
+ selection.request.request_id,
+ );
+ await expect(prepared.publish()).rejects.toThrow("Slow down");
+ expect(await readFile(sibling.path, "utf8")).toBe(siblingBytes);
+ }
+ expect(f.posts).toEqual(
+ order.map((selection) => JSON.stringify(selection.request)),
+ );
+ },
+);
+
+function failCheckpointReads(unreadable: (path: string) => boolean) {
+ const read = fs.readFile;
+ return spyOn(fs, "readFile").mockImplementation((async (
+ ...args: Parameters<typeof read>
+ ) => {
+ if (unreadable(String(args[0])))
+ throw Object.assign(new Error("Unreadable checkpoint"), {
+ code: "EACCES",
+ });
+ return read(...args);
+ }) as typeof read);
+}
+
+test.each(["released", "fresh"])(
+ "a primitive retry uses the lookup for a %s boxed approval without reading unrelated checkpoints",
+ async (kind) => {
+ const { f, prepare, directory, filename, path, request, saved } =
+ await oldScoreApproval("evidence", kind === "released");
+ const boxed = await prepare(new Number(2.5));
+ const approved = boxed.preview.requests[0]!;
+ if (kind === "released")
+ expect(approved.request_id).toBe(request.request_id);
+ f.state.brokenReadback = true;
+ await expect(boxed.publish()).rejects.toThrow("Readback unavailable");
+ const files = await readdir(directory);
+ expect(files.filter((file) => file.endsWith(".pending.json"))).toEqual([
+ filename,
+ ]);
+ const checkpoint = JSON.parse(await readFile(path, "utf8"));
+
+ const unrelated = join(
+ directory,
+ `${hash("sha256", "another-account")}.pending.json`,
+ );
+ await writeFile(
+ unrelated,
+ JSON.stringify({ ...JSON.parse(saved), accountId: "another-account" }),
+ { mode: 0o600 },
+ );
+ using reads = failCheckpointReads((path) => path === unrelated);
+ f.state.brokenReadback = false;
+ const retry = await prepare(2.5);
+ expect(retry.preview.resumed).toBe(true);
+ expect(JSON.stringify(retry.preview.requests)).toBe(
+ JSON.stringify(boxed.preview.requests),
+ );
+ expect((await retry.publish()).receipts[0]!.id).toBe(approved.request_id);
+ expect(f.posts).toEqual([JSON.stringify(approved)]);
+ expect(files).toContain(`${checkpoint.wireKey}.lookup.json`);
+ expect(reads.mock.calls.some(([file]) => String(file) === unrelated)).toBe(
+ false,
+ );
+ },
+);
+
+test("the original boxed approval keeps its filename, request and receipts on a primitive retry", async () => {
+ const { f, prepare, directory, path, filename, request, saved } =
+ await oldScoreApproval("evidence");
+ for (const score of [new Number(9.5), {}]) {
+ await expect(prepare(score)).rejects.toThrow();
+ expect(await readFile(path, "utf8")).toBe(saved);
+ }
+ expect(f.posts).toEqual([]);
+ const recovered = await prepare(new Number(2.5));
+ expect(recovered.preview.resumed).toBe(true);
+ expect(JSON.stringify(recovered.preview.requests)).toBe(
+ JSON.stringify([request]),
+ );
+ f.state.brokenReadback = true;
+ const interrupted = await recovered
+ .publish()
+ .catch((error: unknown) => error);
+ if (!(interrupted instanceof ExternalPublicationError)) throw interrupted;
+ expect(interrupted.result.savedSubmission).toBe(path);
+ const pending = (await readdir(directory)).filter((name) =>
+ name.endsWith(".pending.json"),
+ );
+ expect(pending).toEqual([filename]);
+ expect(JSON.parse(await readFile(path, "utf8")).receipts[0].id).toBe(
+ request.request_id,
+ );
+ f.state.brokenReadback = false;
+ const retry = await prepare(2.5);
+ const sibling = await prepare(2.5);
+ const stale = await prepare(2.5);
+ expect(retry.preview.resumed).toBe(true);
+ expect(JSON.stringify(retry.preview.requests)).toBe(
+ JSON.stringify([request]),
+ );
+ // An older checkpoint at an unknown filename can appear after both retries were prepared.
+ const unknown = join(
+ directory,
+ `${hash("sha256", "other-deployment")}.pending.json`,
+ );
+ const projectionless = JSON.stringify({
+ accountId: boxedNumberCheckpoints.accountId,
+ requests: [request],
+ });
+ await writeFile(unknown, projectionless, { mode: 0o600 });
+ const result = await retry.publish();
+ expect(result.verified).toBe(1);
+ expect(result.receipts.map(({ id }) => id)).toEqual([request.request_id]);
+ {
+ using reads = spyOn(fs, "readFile");
+ expect(await sibling.publish()).toEqual(result);
+ expect(reads.mock.calls.map(([path]) => String(path))).not.toContain(
+ unknown,
+ );
+ }
+ expect(f.posts).toEqual([JSON.stringify(request)]);
+ expect(await Bun.file(path).exists()).toBe(false);
+ expect(await readFile(unknown, "utf8")).toBe(projectionless);
+ await rm(unknown);
+
+ // A later approval of the same input replaced the completed result.
+ const fresh = await prepare(2.5);
+ const replacement = await fresh.publish();
+ expect(replacement.receipts[0]!.id).not.toBe(request.request_id);
+ await expect(stale.publish()).rejects.toThrow(/another publication|review/i);
+ expect(f.posts).toEqual([
+ JSON.stringify(request),
+ JSON.stringify(fresh.preview.requests[0]),
+ ]);
+ expect(await Bun.file(path).exists()).toBe(false);
+});
+
+test.each(["released", "current"])(
+ "a known %s retry preserves its approval when unrelated checkpoints are unreadable",
+ async (format) => {
+ const released = format === "released";
+ const { f, prepare, directory, request, path, saved } =
+ await oldScoreApproval("unreported", released);
+ f.state.throttle = true;
+ for (const score of [1, 2, 3]) {
+ await expect((await prepare(score)).publish()).rejects.toThrow(
+ "Slow down",
+ );
+ }
+ const unrelated = new Set(
+ (await readdir(directory))
+ .filter((name) => name.endsWith(".pending.json"))
+ .map((name) => join(directory, name))
+ .filter((candidate) => candidate !== path),
+ );
+ expect(unrelated.size).toBe(3);
+ let requestId = request.request_id;
+ if (released) {
+ expect(await readFile(path, "utf8")).toBe(saved);
+ } else {
+ const selected = await prepare(2.5);
+ requestId = selected.preview.requests[0]!.request_id;
+ await expect(selected.publish()).rejects.toThrow("Slow down");
+ }
+
+ let unreadable = false;
+ const statePaths = new Set([directory, await fs.realpath(directory)]);
+ using listings = spyOn(fs, "readdir");
+ using reads = failCheckpointReads(
+ (path) => unreadable && unrelated.has(path),
+ );
+ const operations = () => ({
+ listed: listings.mock.calls.some(([path]) =>
+ statePaths.has(String(path)),
+ ),
+ readUnrelated: reads.mock.calls.some(([path]) =>
+ unrelated.has(String(path)),
+ ),
+ });
+ if (released) {
+ const original = await prepare(2.5);
+ reads.mockClear();
+ listings.mockClear();
+ await expect(original.publish()).rejects.toThrow("Slow down");
+ expect(operations()).toEqual({ listed: true, readUnrelated: true });
+ expect(f.posts.at(-1)).toBe(JSON.stringify(request));
+ }
+ unreadable = true;
+ f.state.throttle = false;
+ reads.mockClear();
+ listings.mockClear();
+ const retry = await prepare(2.5);
+ expect(retry.preview.resumed).toBe(true);
+ expect(retry.preview.requests[0]!.request_id).toBe(requestId);
+ expect(operations()).toEqual({ listed: false, readUnrelated: false });
+ f.state.brokenReadback = true;
+ await expect(retry.publish()).rejects.toThrow("Readback unavailable");
+ const stored = JSON.parse(await readFile(path, "utf8"));
+ expect(stored.receipts[0].id).toBe(requestId);
+ if (released)
+ expect(f.posts.slice(-2)).toEqual([
+ JSON.stringify(request),
+ JSON.stringify(request),
+ ]);
+ const posted = [...f.posts];
+ f.state.brokenReadback = false;
+ expect((await (await prepare(2.5)).publish()).receipts).toEqual(
+ stored.receipts,
+ );
+ expect(f.posts).toEqual(posted);
+ expect(operations()).toEqual({ listed: false, readUnrelated: false });
+ listings.mockClear();
+ reads.mockClear();
+ // A fresh selection must discover possible historical approvals.
+ await expect(prepare(4)).rejects.toThrow("Unreadable checkpoint");
+ expect(operations()).toEqual({ listed: true, readUnrelated: true });
+ },
+);
+
+test("a released receipt can finish readback when caching the checkpoint scan fails", async () => {
+ const { f, prepare, path, saved, request } =
+ await oldScoreApproval("unreported");
+ f.state.brokenReadback = true;
+ await expect((await prepare(2.5)).publish()).rejects.toThrow(
+ "Readback unavailable",
+ );
+ const receipts = [f.receipts.get(request.request_id)!];
+ const released = { ...JSON.parse(saved), receipts };
+ await writeFile(path, JSON.stringify(released));
+
+ const rename = fs.rename;
+ using _writes = spyOn(fs, "rename").mockImplementation(async (from, to) => {
+ if (String(to) === path) {
+ const { canonicalChecked, ...pending } = JSON.parse(
+ await readFile(from, "utf8"),
+ );
+ if (canonicalChecked && Bun.deepEquals(pending, released))
+ throw Object.assign(new Error("No space for checkpoint metadata"), {
+ code: "ENOSPC",
+ });
+ }
+ return rename(from, to);
+ });
+ const retry = await prepare(2.5);
+ expect(retry.preview.resumed).toBe(true);
+ const interrupted = await retry.publish().catch((error: unknown) => error);
+ if (!(interrupted instanceof ExternalPublicationError)) throw interrupted;
+ expect(interrupted.result).toMatchObject({
+ status: "interrupted",
+ counts: { updated: 1 },
+ receipts,
+ unacknowledged: 0,
+ savedSubmission: path,
+ });
+ expect(await readFile(path, "utf8")).toBe(JSON.stringify(released));
+ f.state.brokenReadback = false;
+ const completed = await (await prepare(2.5)).publish();
+ expect(completed).toMatchObject({
+ status: "complete",
+ verified: 1,
+ receipts,
+ });
+ expect(f.posts).toEqual([JSON.stringify(request)]);
+});
+
+test("checking a released canonical approval preserves ambiguous boxed report metadata", async () => {
+ const { f, prepare } = await oldScoreApproval("report");
+ f.state.throttle = true;
+ const older = await prepare(2.5, { elapsed: {} });
+ expect(older.preview.resumed).toBe(true);
+ await expect(older.publish()).rejects.toThrow("Slow down");
+ const numeric = await prepare(2.5, { elapsed: 4.5 });
+ expect(numeric.preview.resumed).toBe(false);
+ await expect(numeric.publish()).rejects.toThrow("Slow down");
+ await expect(prepare(2.5, { elapsed: new Number(4.5) })).rejects.toThrow(
+ "older saved import cannot be matched",
+ );
+ expect(f.posts).toEqual([
+ JSON.stringify(older.preview.requests[0]),
+ JSON.stringify(numeric.preview.requests[0]),
+ ]);
+});
+
+test("a second matching checkpoint preserves the receipts shown at preview", async () => {
+ const prior = await oldScoreApproval("unreported");
+ prior.f.state.brokenReadback = true;
+ await expect((await prior.prepare(2.5)).publish()).rejects.toThrow(
+ "Readback unavailable",
+ );
+ const receipts = [prior.f.receipts.get(prior.request.request_id)!];
+ const saved = JSON.stringify({ ...JSON.parse(prior.saved), receipts });
+ await writeFile(prior.path, saved);
+ const held = await prior.prepare(2.5);
+ const competing = await oldScoreApproval("evidence");
+ await competing.prepare(new Number(2.5));
+ const tagged = await readFile(competing.path, "utf8");
+ const otherPath = join(prior.directory, competing.filename);
+ await writeFile(otherPath, tagged);
+
+ const error = await held.publish().catch((cause: unknown) => cause);
+ expect(error).toBeInstanceOf(ExternalPublicationError);
+ if (!(error instanceof ExternalPublicationError)) throw error;
+ expect(error.message).toContain("Two saved imports match");
+ expect(error.result).toMatchObject({
+ status: "interrupted",
+ receipts,
+ counts: { updated: 1 },
+ unacknowledged: 0,
+ });
+ expect(prior.f.posts).toEqual([JSON.stringify(prior.request)]);
+ expect(await readFile(prior.path, "utf8")).toBe(saved);
+ expect(await readFile(otherPath, "utf8")).toBe(tagged);
+});
+
+test("matching released primitive and boxed approvals preserve both requests", async () => {
+ const primitive = await oldScoreApproval("unreported");
+ const boxed = await oldScoreApproval("evidence", false);
+ const held = await primitive.prepare(2.5);
+ const boxedPath = join(primitive.directory, boxed.filename);
+ await writeFile(boxedPath, boxed.saved, { mode: 0o600 });
+ await expect(primitive.prepare(new Number(2.5))).rejects.toThrow(
+ "Two saved imports match",
+ );
+ expect(primitive.f.posts).toEqual([]);
+ expect(await readFile(primitive.path, "utf8")).toBe(primitive.saved);
+ expect(await readFile(boxedPath, "utf8")).toBe(boxed.saved);
+
+ // A later independent canonical approval must still be checked alongside a valid lookup.
+ await rm(primitive.path);
+ await primitive.prepare(new Number(2.5));
+ const tagged = await readFile(boxedPath, "utf8");
+ const lookup = `${JSON.parse(tagged).wireKey}.lookup.json`;
+ expect(await readdir(primitive.directory)).toContain(lookup);
+ await writeFile(primitive.path, primitive.saved, { mode: 0o600 });
+ for (const hintAvailable of [true, false]) {
+ // A process can exit after writing the tagged approval and before saving its lookup.
+ if (!hintAvailable) await rm(join(primitive.directory, lookup));
+ for (const score of [2.5, new Number(2.5)])
+ await expect(primitive.prepare(score)).rejects.toThrow(
+ "Two saved imports match",
+ );
+ await expect(held.publish()).rejects.toThrow("Two saved imports match");
+ expect(await readFile(primitive.path, "utf8")).toBe(primitive.saved);
+ expect(await readFile(boxedPath, "utf8")).toBe(tagged);
+ expect(primitive.f.posts).toEqual([]);
+ }
+
+ // A partial check must see the second approval once it becomes readable.
+ {
+ using _reads = failCheckpointReads((path) => path === boxedPath);
+ primitive.f.state.throttle = true;
+ await expect(held.publish()).rejects.toThrow("Slow down");
+ }
+ await expect(primitive.prepare(2.5)).rejects.toThrow(
+ "Two saved imports match",
+ );
+ expect(await readFile(primitive.path, "utf8")).toBe(primitive.saved);
+ expect(await readFile(boxedPath, "utf8")).toBe(tagged);
+ expect(primitive.f.posts).toEqual([JSON.stringify(primitive.request)]);
+});
+
+test.each(["report", "mixed", "aliased"])(
+ "ambiguous historical %s data preserves the pending approval",
+ async (field) => {
+ const { f, prepare, path, saved } = await oldScoreApproval(field);
+ const retry =
+ field === "mixed"
+ ? prepare(new Number(2.5), { elapsed: {} })
+ : prepare(2.5, { elapsed: new Number(4.5) });
+ await expect(retry).rejects.toThrow();
+ expect(await readFile(path, "utf8")).toBe(saved);
+ expect(f.posts).toEqual([]);
+ },
+);
+
+test("a new boxed report does not claim an unrelated older approval", async () => {
+ const { f, prepare, path, saved, request } =
+ await oldScoreApproval("unreported");
+ const fresh = await prepare(2.5, { elapsed: new Number(4.5) });
+ expect(fresh.preview.resumed).toBe(false);
+ expect(fresh.preview.requests[0]!.request_id).not.toBe(request.request_id);
+ expect(JSON.stringify((await prepare(2.5)).preview.requests)).toBe(
+ JSON.stringify([request]),
+ );
+ expect(await readFile(path, "utf8")).toBe(saved);
+ expect(f.posts).toEqual([]);
+});
+
+test("an older boxed score with an empty report resumes its original request", async () => {
+ const { f, prepare, request } = await oldScoreApproval("empty-report");
+ const boxed = await prepare(new Number(12), {});
+ expect(boxed.preview.resumed).toBe(true);
+ f.state.throttle = true;
+ await expect(boxed.publish()).rejects.toThrow("Slow down");
+ f.state.throttle = false;
+ const primitive = await prepare(12, {});
+ expect(primitive.preview.resumed).toBe(true);
+ expect((await primitive.publish()).verified).toBe(1);
+ const approved = JSON.stringify(request);
+ expect(f.posts).toEqual([approved, approved]);
+});
+
+test("a boxed false flag remains independent of an approved empty object", async () => {
+ const f = await fixture();
+ const prepare = (score: unknown, flag: unknown) => {
+ const input = scoredFindings(score);
+ Object.assign(input.findings[0]!.evidence.source_data, { flag });
+ return prepareNormalizedExternalPublication(input, options, f.deps);
+ };
+ f.state.throttle = true;
+ const original = await prepare(1, {});
+ await expect(original.publish()).rejects.toThrow("Slow down");
+ const boxed = await prepare(new Number(1), new Boolean(false));
+ expect(boxed.preview.resumed).toBe(false);
+ expect(boxed.preview.requests[0]!.request_id).not.toBe(
+ original.preview.requests[0]!.request_id,
+ );
+ await expect(boxed.publish()).rejects.toThrow("Slow down");
+ expect(JSON.parse(f.posts[1]!).items[0].evidence.source_data).toEqual({
+ score: 1,
+ flag: false,
+ });
+ const primitive = await prepare(1, false);
+ expect(primitive.preview.resumed).toBe(true);
+ f.state.throttle = false;
+ expect((await primitive.publish()).receipts[0]!.id).toBe(
+ boxed.preview.requests[0]!.request_id,
+ );
+ expect(f.posts[2]).toBe(f.posts[1]);
+ const empty = await prepare(1, {});
+ expect(empty.preview.resumed).toBe(true);
+ expect(JSON.stringify(empty.preview.requests[0])).toBe(f.posts[0]!);
+});
+
+test("an older empty flag cannot authorize a boxed Boolean", async () => {
+ const { f, prepare, path, saved, request } = await oldScoreApproval("flag");
+ await expect(
+ prepare(new Number(12), undefined, new Boolean(true)),
+ ).rejects.toThrow();
+ expect(await readFile(path, "utf8")).toBe(saved);
+ expect(f.posts).toEqual([]);
+ const original = await prepare(new Number(12), undefined, {});
+ expect(original.preview.resumed).toBe(true);
+ expect((await original.publish()).verified).toBe(1);
+ expect(f.posts).toEqual([JSON.stringify(request)]);
+});
+
+test.each(["evidence", "report"])(
+ "a fresh boxed %s reuses its historical filename until the numeric retry completes",
+ async (field) => {
+ const { f, prepare, path } = await oldScoreApproval(field, false);
+ const select = (value: unknown) =>
+ field === "report" ? prepare(2.5, { elapsed: value }) : prepare(value);
+ const value = field === "report" ? 4.5 : 2.5;
+ const ordinary = await select({});
+ const competing = await select(new Number(value + 1));
+ const boxed = await select(new Number(value));
+ f.state.throttle = true;
+ await expect(boxed.publish()).rejects.toThrow("Slow down");
+ const saved = await readFile(path, "utf8");
+ const body = f.posts[0]!;
+ expect(JSON.parse(saved).requests).toEqual([JSON.parse(body)]);
+ for (const stale of [ordinary, competing]) {
+ await expect(stale.publish()).rejects.toThrow(/pending|review/i);
+ }
+ await expect(select({})).rejects.toThrow(/finish the other pending/i);
+ expect(f.posts).toEqual([body]);
+ expect(await readFile(path, "utf8")).toBe(saved);
+
+ const numeric = await select(value);
+ expect(numeric.preview.resumed).toBe(true);
+ expect(numeric.preview.requests).toEqual([JSON.parse(body)]);
+ f.state.throttle = false;
+ expect((await numeric.publish()).verified).toBe(1);
+ expect(await Bun.file(path).exists()).toBe(false);
+ const fresh = await select({});
+ expect(fresh.preview.resumed).toBe(false);
+ expect(fresh.preview.requests[0]!.request_id).not.toBe(
+ numeric.preview.requests[0]!.request_id,
+ );
+ expect((await fresh.publish()).verified).toBe(1);
+ expect(f.posts).toEqual([
+ body,
+ body,
+ JSON.stringify(fresh.preview.requests[0]),
+ ]);
+ },
+);
+
+test("normalized SDK publication rejects nonfinite raw evidence without JSON coercion", async () => {
+ const f = await fixture();
+ const input = {
+ read: 1,
+ findings: [
+ {
+ source_finding_id: "nonfinite-raw-evidence",
+ evidence: {
+ title: "Synthetic evidence",
+ severity: "high" as const,
+ source_data: { score: Infinity },
+ },
+ },
+ ],
+ excluded: [],
+ };
+ await expect(
+ prepareNormalizedExternalPublication(input, options, f.deps),
+ ).rejects.toThrow("finite JSON numbers");
+ expect(input.findings[0]!.evidence.source_data.score).toBe(Infinity);
+ expect(f.posts).toHaveLength(0);
+});
+
+test("normalized SDK publication rejects unserializable report metadata before upload", async () => {
+ const f = await fixture();
+ const input = {
+ read: 1,
+ report: { elapsedNs: 1n },
+ findings: [
+ {
+ source_finding_id: "synthetic-report",
+ evidence: { title: "Example", severity: "unknown" as const },
+ },
+ ],
+ excluded: [],
+ };
+ await expect(
+ prepareNormalizedExternalPublication(input, options, f.deps),
+ ).rejects.toThrow(/bigint/i);
+ await expect(
+ prepareNormalizedExternalPublication(
+ { ...input, report: { elapsed: new Number(Infinity) } },
+ options,
+ f.deps,
+ ),
+ ).rejects.toThrow("finite JSON numbers");
+ await expect(
+ prepareNormalizedExternalPublication(
+ { ...input, report: { toJSON: () => Infinity } },
+ options,
+ f.deps,
+ ),
+ ).rejects.toThrow("finite JSON numbers");
+ expect(f.posts).toHaveLength(0);
+});
+
+test.each([Error, TypeError])(
+ "a native %p resumes its released message-only approval",
+ async (Diagnostic) => {
+ const { f, directory, prepare, request } = await oldScoreApproval(
+ "evidence",
+ false,
+ );
+ request.request_id = "15ab6427-635d-4537-9975-f7f536155b49";
+ const { path, saved } = await stageReleasedCheckpoint(f, {
+ filename:
+ "19d4eb5e5738ac54ae3d847f50534f09ac60882b0145fa5abc4260299e7f02b6.pending.json",
+ requests: [request],
+ inputDigest:
+ "603ea14278affa7a9a27fc2a624f5a2ce8fd85460f99175bef69a0572430b59b",
+ savedEvidenceDigest:
+ "875a8aec2f1b7c573a6d05d983f6afa0f964a6e1c96d22c8c8691cc2de824ccc",
+ });
+ const diagnostic = () =>
+ Object.assign(
+ new Diagnostic("scanner message", { cause: "scanner failure" }),
+ { code: "E_TEST" },
+ );
+ const exportedDiagnostic = Object.assign(diagnostic(), {
+ toJSON: () => ({ code: "TIMEOUT" }),
+ });
+ const prepareExported = () =>
+ prepare(2.5, { errors: [exportedDiagnostic] });
+ await expect(prepareExported()).rejects.toThrow("cannot be matched");
+ expect(await readFile(path, "utf8")).toBe(saved);
+ expect(f.posts).toEqual([]);
+
+ const uncertain = diagnostic();
+ let stackReads = 0;
+ Object.defineProperty(uncertain, "stack", {
+ get() {
+ stackReads++;
+ return "custom stack";
+ },
+ });
+ await expect(prepare(2.5, { errors: [uncertain] })).rejects.toThrow(
+ "cannot be matched",
+ );
+ expect(stackReads).toBe(0);
+ expect(f.posts).toEqual([]);
+ expect(await readFile(path, "utf8")).toBe(saved);
+
+ const prepareUnsafeExport = (error: Error, beforeExport?: () => void) => {
+ const input = scoredFindings(2.5, { errors: [error] });
+ Object.setPrototypeOf(input.findings[0]!.evidence.source_data, {
+ toJSON() {
+ beforeExport?.();
+ return { score: 2 ** 53 };
+ },
+ });
+ return prepareReleased(f, input);
+ };
+ const changed = diagnostic();
+ await expect(
+ prepareUnsafeExport(changed, () =>
+ Object.defineProperty(changed, "stack", {
+ get() {
+ stackReads++;
+ return "custom stack";
+ },
+ }),
+ ),
+ ).rejects.toThrow("safe integer");
+ expect(stackReads).toBe(0);
+ expect(f.posts).toEqual([]);
+ expect(await readFile(path, "utf8")).toBe(saved);
+
+ const report = { errors: [diagnostic()] };
+ expect((await prepare(2.5, report)).preview.resumed).toBe(true);
+ const retry = await prepareUnsafeExport(report.errors[0]!);
+ expect(retry.preview.resumed).toBe(true);
+ const completed = await retry.publish();
+ expect(completed.receipts[0]!.id).toBe(request.request_id);
+ const expected = structuredClone(report.errors[0]!);
+ for (const value of [retry.preview.report, completed.report]) {
+ const error = (value?.["errors"] as Error[])[0]!;
+ expect(error).toBeInstanceOf(Diagnostic);
+ expect(error.message).toBe(expected.message);
+ expect(error.stack).toBe(expected.stack);
+ expect(error.cause).toBe(expected.cause);
+ expect(error).not.toHaveProperty("code");
+ }
+ const results = await Promise.all(
+ (await readdir(directory))
+ .filter((name) => name.endsWith(".result.json"))
+ .map(
+ async (name) =>
+ JSON.parse(
+ await readFile(join(directory, name), "utf8"),
+ ) as ExternalPublicationResult,
+ ),
+ );
+ const receipt = results.find(
+ (result) => result.receipts[0]?.id === request.request_id,
+ );
+ expect(receipt?.report).toEqual({
+ errors: [{ message: "scanner message" }],
+ });
+ expect(f.posts).toEqual([JSON.stringify(request)]);
+ expect((await prepareExported()).preview.report?.["errors"]).toEqual([
+ { code: "TIMEOUT" },
+ ]);
+ const fresh = await prepare(2.5, report);
+ expect(fresh.preview.resumed).toBe(false);
+ expect(fresh.preview.report?.["errors"]).toEqual([
+ expect.objectContaining({ message: "scanner message", code: "E_TEST" }),
+ ]);
+ },
+);
+
+test("publication JSON retains native Error messages and scanner diagnostics", async () => {
+ const scannerErrors = opengrepJson().errors;
+ const error = Object.assign(
+ runInNewContext(
+ 'new TypeError(undefined, { cause: "scanner failure" })',
+ ) as TypeError,
+ { code: "RULE_TIMEOUT", message: "Scanner rule timed out." },
+ );
+ const nested = {
+ path: "src/example.ts",
+ cause: new Error("Parser stopped."),
+ };
+ const report = { errors: [error, nested, ...scannerErrors] };
+ const jsonErrors = [
+ { message: error.message, code: error.code },
+ { path: nested.path, cause: { message: nested.cause.message } },
+ ...scannerErrors,
+ ];
+ const f = await fixture();
await expect(
prepareNormalizedExternalPublication(
{ read: 0, findings: [], excluded: [], report },
@@ -4177,12 +5933,16 @@ test("publication JSON retains native Error messages and scanner diagnostics", a
options,
f.deps,
);
- const original = structuredClone(report);
nested.cause.message = "Caller changed the diagnostic.";
const completed = await prepared.publish();
expect(completed.verified).toBe(1);
for (const value of [prepared.preview.report, completed.report]) {
- expect(value).toEqual(original);
+ const errors = value?.["errors"] as Error[];
+ expect(errors[0]).toBeInstanceOf(TypeError);
+ expect(errors[0]).toHaveProperty("code", error.code);
+ expect(errors[0]!.message).toBe(error.message);
+ expect(errors[0]!.cause).toBe(structuredClone(error).cause);
+ expect(errors[1]?.cause).toHaveProperty("message", "Parser stopped.");
expect(value?.["errors"]).toEqual([
expect.any(Error),
{ path: nested.path, cause: expect.any(Error) },
@@ -5121,3 +6881,650 @@ test("publication rejects malformed options without writing findings", async ()
expect(JSON.parse(preview.stdout.text())).toMatchObject({ dryRun: true });
expect(f.posts).toHaveLength(0);
});
+
+test("a held primitive retry cannot republish after a newer boxed approval", async () => {
+ const f = await fixture();
+ const prepare = (elapsed: unknown) =>
+ prepareNormalizedExternalPublication(
+ scoredFindings(12, { elapsed }),
+ options,
+ f.deps,
+ );
+ const original = await prepare(2.5);
+ f.state.loseResponse = true;
+ await expect(original.publish()).rejects.toThrow(/resume the saved request/);
+ const held = await prepare(2.5);
+ expect(held.preview.resumed).toBe(true);
+ const completed = await (await prepare(2.5)).publish();
+
+ const newer = await prepare(new Number(2.5));
+ const newerId = newer.preview.requests[0]!.request_id;
+ expect(newerId).not.toBe(original.preview.requests[0]!.request_id);
+ f.state.loseResponse = true;
+ await expect(newer.publish()).rejects.toThrow(/resume the saved request/);
+ const before = [...f.posts];
+ expect(await held.publish()).toEqual(completed);
+ expect(f.posts).toEqual(before);
+
+ const remaining = await prepare(2.5);
+ expect(remaining.preview.requests).toEqual(newer.preview.requests);
+ expect((await remaining.publish()).receipts[0]!.id).toBe(newerId);
+ const directory = join(f.root, "state", "external-finding-publications");
+ const filename = (await readdir(directory)).find((name) =>
+ name.endsWith(".result.json"),
+ )!;
+ const resultPath = join(directory, filename);
+ const saved = await readFile(resultPath, "utf8");
+ await expect(held.publish()).rejects.toThrow(/changed after preview/);
+ expect(await readFile(resultPath, "utf8")).toBe(saved);
+ expect(f.posts).toEqual([
+ ...before,
+ JSON.stringify(newer.preview.requests[0]),
+ ]);
+});
+
+const concurrentBoxedApproval =
+ "concurrent prepares reuse only the matching completed boxed approval";
+test(concurrentBoxedApproval, async () => {
+ const { runTestInSubprocess } = await import("./support/test-subprocess.js");
+ if (runTestInSubprocess(import.meta.filename, concurrentBoxedApproval))
+ return;
+
+ for (const state of [
+ "original result",
+ "newer pending",
+ "newer complete",
+ "released result",
+ "released different",
+ "released malformed",
+ ]) {
+ const { f, prepare, directory, path, request } =
+ await oldScoreApproval("evidence");
+ const atLock = Promise.withResolvers<void>();
+ const release = Promise.withResolvers<void>();
+ const lstat = fs.lstat;
+ let holdFirst = true;
+ const metadata = spyOn(fs, "lstat").mockImplementation((async (
+ ...args: Parameters<typeof fs.lstat>
+ ) => {
+ const file = String(args[0]);
+ if (
+ holdFirst &&
+ file.startsWith(directory) &&
+ file.endsWith(".lock.sqlite")
+ ) {
+ holdFirst = false;
+ atLock.resolve();
+ await release.promise;
+ }
+ return lstat(...args);
+ }) as typeof fs.lstat);
+
+ const waiting = prepare(new Number(2.5));
+ try {
+ await Promise.race([atLock.promise, waiting]);
+ const leader = await prepare(new Number(2.5));
+ const { wireKey } = JSON.parse(await readFile(path, "utf8"));
+ const completed = await leader.publish();
+ expect(completed.receipts[0]!.id).toBe(request.request_id);
+ expect(f.posts).toEqual([JSON.stringify(request)]);
+ let pending: { path: string; bytes: string } | undefined;
+ if (state.startsWith("released")) {
+ // The released publisher stores this receipt under its original checkpoint filename.
+ const oldResult = path.replace(/\.pending\.json$/, ".result.json");
+ const currentName = `${wireKey}.result.json`;
+ const seed = (await readdir(directory)).find(
+ (name) => name.endsWith(".result.json") && name !== currentName,
+ )!;
+ const currentResult = join(directory, currentName);
+ await fs.rename(currentResult, oldResult);
+ if (state === "released malformed") {
+ const result = JSON.parse(await readFile(oldResult, "utf8"));
+ result.receipts[0].results = [];
+ await writeFile(oldResult, JSON.stringify(result));
+ } else if (state === "released different") {
+ await fs.copyFile(join(directory, seed), oldResult);
+ } else {
+ // A different completed request can still occupy the current result filename.
+ await fs.copyFile(join(directory, seed), currentResult);
+ }
+ } else if (state !== "original result") {
+ const newer = await prepare(2.5);
+ const newerId = newer.preview.requests[0]!.request_id;
+ expect(newerId).not.toBe(request.request_id);
+ if (state === "newer pending") {
+ f.state.throttle = true;
+ await expect(newer.publish()).rejects.toThrow("Slow down");
+ const [name] = (await readdir(directory)).filter((name) =>
+ name.endsWith(".pending.json"),
+ );
+ const path = join(directory, name!);
+ pending = { path, bytes: await readFile(path, "utf8") };
+ expect(JSON.parse(pending.bytes).requests[0].request_id).toBe(
+ newerId,
+ );
+ f.state.throttle = false;
+ } else {
+ expect((await newer.publish()).receipts[0]!.id).toBe(newerId);
+ }
+ }
+ const before = [...f.posts];
+ release.resolve();
+ if (state === "released malformed") {
+ await expect(waiting).rejects.toThrow();
+ } else if (state === "newer complete" || state === "released different") {
+ await expect(waiting).rejects.toThrow(/changed while preparing/);
+ } else {
+ const trailing = await waiting;
+ expect(trailing.preview.resumed).toBe(true);
+ expect(trailing.preview.requests[0]!.request_id).toBe(
+ request.request_id,
+ );
+ expect(await trailing.publish()).toEqual(completed);
+ }
+ if (pending)
+ expect(await readFile(pending.path, "utf8")).toBe(pending.bytes);
+ expect(f.posts).toEqual(before);
+ } finally {
+ release.resolve();
+ metadata.mockRestore();
+ await Promise.allSettled([waiting]);
+ }
+ }
+});
+
+test("a lookup hint cannot substitute for the tagged approval", async () => {
+ const { f, prepare, directory, path, filename, request } =
+ await oldScoreApproval("evidence");
+ f.state.throttle = true;
+ await expect((await prepare(new Number(2.5))).publish()).rejects.toThrow(
+ "Slow down",
+ );
+ const tag = JSON.parse(await readFile(path, "utf8"));
+ const lookup = join(directory, `${tag.wireKey}.lookup.json`);
+ const suffix = ".pending.json";
+ expect(JSON.parse(await readFile(lookup, "utf8"))).toBe(
+ filename.slice(0, -suffix.length),
+ );
+
+ const otherReport = { units: "seconds" };
+ const other = await prepare(new Number(3.5), otherReport);
+ await expect(other.publish()).rejects.toThrow("Slow down");
+ const [otherName] = (await readdir(directory)).filter(
+ (name) => name.endsWith(suffix) && name !== filename,
+ );
+ const otherPath = join(directory, otherName!);
+ const otherSaved = await readFile(otherPath, "utf8");
+ const otherTag = JSON.parse(otherSaved);
+ expect(otherTag.wireKey).not.toBe(tag.wireKey);
+ const otherKey = otherName!.slice(0, -suffix.length);
+ expect(
+ JSON.parse(
+ await readFile(
+ join(directory, `${otherTag.wireKey}.lookup.json`),
+ "utf8",
+ ),
+ ),
+ ).toBe(otherKey);
+ f.posts.length = 0;
+ f.state.throttle = false;
+
+ const outside = join(directory, "..", "foreign.pending.json");
+ await writeFile(outside, otherSaved, { mode: 0o600 });
+ const reads = spyOn(fs, "readFile");
+ try {
+ for (const hint of [
+ undefined,
+ "{",
+ JSON.stringify("../foreign"),
+ JSON.stringify(otherKey),
+ ]) {
+ if (hint === undefined) await rm(lookup);
+ else await writeFile(lookup, hint);
+ const actual = await prepare(2.5);
+ expect(actual.preview.resumed).toBe(true);
+ expect(actual.preview.requests[0]!.request_id).toBe(request.request_id);
+ }
+ expect(reads.mock.calls.map(([path]) => String(path))).not.toContain(
+ outside,
+ );
+ } finally {
+ reads.mockRestore();
+ }
+ const completed = await (await prepare(2.5)).publish();
+ expect(completed.receipts[0]!.id).toBe(request.request_id);
+ expect(f.posts).toEqual([JSON.stringify(request)]);
+ expect(await readFile(otherPath, "utf8")).toBe(otherSaved);
+
+ const held = await prepare(3.5, otherReport);
+ expect(held.preview.requests[0]!.request_id).toBe(
+ other.preview.requests[0]!.request_id,
+ );
+ await rm(otherPath);
+ await expect(held.publish()).rejects.toThrow(/changed after preview/);
+ const fresh = await prepare(3.5, otherReport);
+ expect(fresh.preview.resumed).toBe(false);
+ expect(fresh.preview.requests[0]!.request_id).not.toBe(
+ other.preview.requests[0]!.request_id,
+ );
+ expect(f.posts).toEqual([JSON.stringify(request)]);
+});
+
+test("a lookup write or cleanup failure leaves the approved request usable", async () => {
+ const { f, prepare, path, directory, request } =
+ await oldScoreApproval("evidence");
+ const rename = fs.rename;
+ const write = spyOn(fs, "rename").mockImplementation(async (from, to) => {
+ if (String(to).endsWith(".lookup.json"))
+ throw new Error("Lookup write failed");
+ return rename(from, to);
+ });
+ f.state.throttle = true;
+ try {
+ const boxed = await prepare(new Number(2.5));
+ await expect(boxed.publish()).rejects.toThrow("Slow down");
+ expect(
+ write.mock.calls.some(([, to]) => String(to).endsWith(".lookup.json")),
+ ).toBe(true);
+ } finally {
+ write.mockRestore();
+ }
+ const tag = JSON.parse(await readFile(path, "utf8"));
+ const lookup = join(directory, `${tag.wireKey}.lookup.json`);
+ expect(await Bun.file(lookup).exists()).toBe(false);
+ f.state.throttle = false;
+ const remove = fs.rm;
+ const cleanup = spyOn(fs, "rm").mockImplementation(async (file, options) => {
+ if (String(file) === lookup) throw new Error("Lookup cleanup failed");
+ return remove(file, options);
+ });
+ try {
+ const retry = await prepare(2.5);
+ expect(retry.preview.resumed).toBe(true);
+ const result = await retry.publish();
+ expect(result.status).toBe("complete");
+ expect(result.verified).toBe(1);
+ expect(result.receipts[0]!.id).toBe(request.request_id);
+ expect(cleanup.mock.calls.some(([file]) => String(file) === lookup)).toBe(
+ true,
+ );
+ } finally {
+ cleanup.mockRestore();
+ }
+ expect(await Bun.file(path).exists()).toBe(false);
+ expect(await Bun.file(lookup).exists()).toBe(true);
+ expect((await prepare(2.5)).preview.resumed).toBe(false);
+ expect(f.posts).toEqual([JSON.stringify(request), JSON.stringify(request)]);
+});
+
+test("partially unboxing an approved input resumes its verified wire request", async () => {
+ const { f, prepare, directory, request } = await oldScoreApproval(
+ "evidence",
+ false,
+ );
+ // This filename, projection and request ID came from the released SDK with b boxed.
+ request.request_id = "dd905f37-1437-44a3-b559-8c88fe873e52";
+ Object.assign(request.items[0]!.evidence.source_data, {
+ score: { a: 7, b: 99 },
+ });
+ const { path: oldPath, saved } = await stageReleasedCheckpoint(f, {
+ filename:
+ "ad0ce218fdfcfa75455f2fbda30dc6e248a79419190e5fa77a7c12005ead7926.pending.json",
+ requests: [request],
+ inputDigest:
+ "232fbafeda24e0939a033a0d59479908389a8e167612feabc174e36f716303a5",
+ savedEvidenceDigest:
+ "3a5021a32dc41a1fe4b2d8e2f74abfd1238ef63cf5bfb4cf44cb28bb09e5bfad",
+ });
+ const partial = () => prepare({ a: 7, b: new Number(2) });
+ await expect(partial()).rejects.toThrow("cannot be matched");
+
+ const approved = await prepare({ a: new Number(7), b: new Number(2) });
+ f.state.throttle = true;
+ await expect(approved.publish()).rejects.toThrow("Slow down");
+ f.state.throttle = false;
+ const body = JSON.stringify(approved.preview.requests[0]);
+ const hinted = await partial();
+ expect(hinted.preview.resumed).toBe(true);
+ expect(JSON.stringify(hinted.preview.requests[0])).toBe(body);
+ const lookup = (await readdir(directory)).find((name) =>
+ name.endsWith(".lookup.json"),
+ )!;
+ await rm(join(directory, lookup));
+ const retry = await partial();
+ expect(retry.preview.resumed).toBe(true);
+ expect(JSON.stringify(retry.preview.requests[0])).toBe(body);
+ expect((await retry.publish()).receipts[0]!.id).toBe(
+ approved.preview.requests[0]!.request_id,
+ );
+ expect(f.posts).toEqual([body, body]);
+ expect(await readFile(oldPath, "utf8")).toBe(saved);
+});
+
+test("preparing across a checkpoint handoff returns the observed approval", async () => {
+ const {
+ f,
+ prepare,
+ path: boxedPath,
+ } = await oldScoreApproval("evidence", false);
+ const first = await prepare(2.5);
+ f.state.loseResponse = true;
+ const interrupted = await first.publish().catch((error: unknown) => error);
+ if (!(interrupted instanceof ExternalPublicationError)) throw interrupted;
+ const canonicalPath = interrupted.result.savedSubmission!;
+ const leading = await prepare(2.5);
+
+ const captured = Promise.withResolvers<void>();
+ const release = Promise.withResolvers<void>();
+ let hold = true;
+ const read = fs.readFile;
+ const reads = spyOn(fs, "readFile").mockImplementation((async (
+ ...args: Parameters<typeof read>
+ ) => {
+ const bytes = await read(...args);
+ if (hold && String(args[0]) === canonicalPath) {
+ hold = false;
+ captured.resolve();
+ await release.promise;
+ }
+ return bytes;
+ }) as typeof read);
+ const waiting = prepare(2.5);
+ try {
+ await Promise.race([captured.promise, waiting]);
+ expect(hold).toBe(false);
+ const completed = await leading.publish();
+ expect(await Bun.file(canonicalPath).exists()).toBe(false);
+ const next = await prepare(new Number(2.5));
+ expect(next.preview.requests[0]!.request_id).not.toBe(
+ first.preview.requests[0]!.request_id,
+ );
+ f.state.throttle = true;
+ await expect(next.publish()).rejects.toThrow("Slow down");
+ const newer = await readFile(boxedPath, "utf8");
+ const before = [...f.posts];
+ release.resolve();
+ const trailing = await waiting;
+ expect(trailing.preview.resumed).toBe(true);
+ expect(trailing.preview.requests).toEqual(first.preview.requests);
+ expect(await trailing.publish()).toEqual(completed);
+ expect(await readFile(boxedPath, "utf8")).toBe(newer);
+ expect(f.posts).toEqual(before);
+ } finally {
+ release.resolve();
+ reads.mockRestore();
+ await Promise.allSettled([waiting]);
+ }
+});
+
+test("a changing GitHub ref cannot expand an older pending publication", async () => {
+ const f = await fixture();
+ const prepare = (
+ selected: ReturnType<typeof normalizeGitHubCodeScanningAlerts>,
+ ) =>
+ prepareNormalizedExternalPublication(
+ selected,
+ { ...options, provider: "github" },
+ f.deps,
+ );
+ const first = importedAlert();
+ const preview = (
+ await prepare(normalizeGitHubCodeScanningAlerts([first], "cloud"))
+ ).preview;
+ // The released reader excluded the second alert before its rule serializer ran.
+ const { path, saved } = await stageReleasedCheckpoint(f, {
+ filename:
+ "0b43d13c77ddf5fa43ce0bb536402595aea270191adde5974f2206753fa607b1.pending.json",
+ accountId: preview.accountId,
+ requests: [
+ {
+ ...preview.requests[0]!,
+ request_id: "f49f8193-10b0-4c6c-a20d-390304a5f8d0",
+ },
+ ],
+ inputDigest:
+ "af139e1baa9e9e70a47d4ef0f811cb3960e9c280252affab51c250e7e6f9aa88",
+ });
+ expect(hash("sha256", saved)).toBe(
+ "5f17c21d058c685c128d488b5a2a1e466fa5d18e37cc5284268b95ef25307293",
+ );
+
+ const second = importedAlert();
+ second.number = second.alert.number = 13;
+ second.url = second.alert.html_url = first.url.replace(/12$/, "13");
+ let ref = "";
+ Object.defineProperty(second.alert.most_recent_instance, "ref", {
+ enumerable: true,
+ get: () => ref,
+ });
+ Object.defineProperty(second.alert.rule, "toJSON", {
+ value() {
+ ref = "refs/heads/changed";
+ return { ...this };
+ },
+ });
+ const selected = normalizeGitHubCodeScanningAlerts([first, second], "cloud");
+ expect(selected.findings).toHaveLength(2);
+ await expect(prepare(selected)).rejects.toThrow("cannot be matched");
+ expect(f.posts).toHaveLength(0);
+ expect(await readFile(path, "utf8")).toBe(saved);
+});
+
+test("a normalized Buffer at the Cloud depth limit resumes only its released request", async () => {
+ const f = await fixture();
+ const prepare = (leaf: unknown) => {
+ let tree = leaf;
+ for (let i = 0; i < 17; i++) tree = { nested: tree };
+ const selected = scoredFindings(new Number(2.5));
+ Object.assign(selected.findings[0]!.evidence.source_data, { tree });
+ return prepareReleased(f, selected);
+ };
+ await expect(prepare(Buffer.from([1]))).rejects.toThrow(
+ "Cloud JSON depth limit of 20",
+ );
+
+ // The released SDK cloned this Buffer into a Uint8Array with one fewer JSON level.
+ const request = {
+ ...(await prepare({ 0: 1 })).preview.requests[0]!,
+ request_id: "54bf2d2b-9380-415e-a364-963088768421",
+ };
+ const serialized = JSON.stringify(request);
+ expect(hash("sha256", serialized)).toBe(
+ "5d105b251af91c09d0b206404bd9332031da8e3cb4bb02a9b8add4e4c21814c4",
+ );
+ const { path, saved } = await stageReleasedCheckpoint(f, {
+ filename:
+ "8376bdd00875ba8349ea341fab1c1d7c91eface0f2346d5eacf28e5c534a6d6a.pending.json",
+ requests: [request],
+ inputDigest:
+ "4eace8a1f10136be4258341e48f747f3609919238018ad5f8d84ceebfe469bb2",
+ savedEvidenceDigest:
+ "68702d81bc70bb23815207a069b0780b2b5b1f304250ca4696ecfe129c48e497",
+ });
+ await expect(prepare(Buffer.from([2]))).rejects.toThrow();
+ expect(await readFile(path, "utf8")).toBe(saved);
+ expect(f.posts).toHaveLength(0);
+
+ const recovered = await prepare(Buffer.from([1]));
+ expect(recovered.preview.resumed).toBe(true);
+ expect((await recovered.publish()).receipts[0]!.id).toBe(request.request_id);
+ expect(f.posts).toEqual([serialized]);
+});
+
+test("the local GitHub reader recovers its released Buffer at the Cloud depth limit", async () => {
+ const f = await fixture();
+ const selected = (leaf: unknown) => {
+ const row = importedAlert();
+ let payload = leaf;
+ for (let i = 0; i < 16; i++) payload = { nested: payload };
+ Object.assign(row.alert, { payload });
+ return normalizeGitHubCodeScanningAlerts([row], "local");
+ };
+ const prepare = (input: ReturnType<typeof selected>) =>
+ prepareReleased(f, input, { ...options, provider: "github" });
+ await expect(prepare(selected(Buffer.from([1])))).rejects.toThrow(
+ "Cloud JSON depth limit of 20",
+ );
+
+ // The released publisher sent the native byte without the Buffer's JSON envelope.
+ const request = {
+ ...(await prepare(selected({ 0: 1 }))).preview.requests[0]!,
+ request_id: "3681fc33-491a-4ef4-877c-4cdd6f7dac6c",
+ };
+ const wire = JSON.stringify(request);
+ expect(hash("sha256", wire)).toBe(
+ "0603739963094b95e0f454f2cfd70ec9ee9f7abf891d43d536a67ad27dcf7b95",
+ );
+ const { path, saved } = await stageReleasedCheckpoint(f, {
+ filename:
+ "0f045c3b26dc68db90c2d4e8462be50efa7a758d256398e22f13d1fe070e552a.pending.json",
+ requests: [request],
+ inputDigest:
+ "85488c56d0f48bb7dff1f931e69603aa4aa75c1ef287bf612fd84dc7df369aa2",
+ });
+
+ const edited = selected(Buffer.from([1]));
+ edited.findings[0]!.evidence.title = "Edited after the local read";
+ await expect(prepare(edited)).rejects.toThrow("Cloud JSON depth limit of 20");
+ expect(await readFile(path, "utf8")).toBe(saved);
+ expect(f.posts).toHaveLength(0);
+
+ const resumed = await prepare(selected(Buffer.from([1])));
+ expect(resumed.preview.resumed).toBe(true);
+ expect((await resumed.publish()).receipts[0]!.id).toBe(request.request_id);
+ expect(f.posts).toEqual([wire]);
+});
+
+test.each([
+ [
+ "cloned",
+ ["main"],
+ "0e174a52fb4c08c30d5b452239859f60186b2e5f3ea11476b7c174712b5f9dca",
+ "f97a11da51b2461c09e003c6167239198cde1e28a0b0ec97d8e40a512f396c99",
+ ],
+ [
+ "reversed",
+ ["a", "b"],
+ "a64d3d2ca1432fae7c53f2880f7e6719f4e4f673ce61e85f8084980d95bc55ae",
+ "28c71167a82f47a1f038e6469f36a8f740fe3584ba67f8d4a52d5dbcbdff44d5",
+ ],
+] as const)(
+ "preserves a released GitHub Buffer checkpoint with %s reader output",
+ async (kind, refs, key, digest) => {
+ const f = await fixture();
+ const prepare = (bytes: unknown) => {
+ const rows = refs.map((ref) => {
+ const row = importedAlert(`refs/heads/${ref}`);
+ Object.assign(row.alert, { bytes });
+ return row;
+ });
+ const result = normalizeGitHubCodeScanningAlerts(rows);
+ if (kind === "reversed") result.findings.reverse();
+ return prepareReleased(
+ f,
+ kind === "cloned" ? structuredClone(result) : result,
+ { ...options, provider: "github" },
+ );
+ };
+ // The released reader cloned each Buffer into a Uint8Array before publication.
+ const requests = (await prepare({ 0: 1 })).preview.requests;
+ const { checkpoint, directory, path, saved } =
+ await stageReleasedCheckpoint(f, {
+ filename: `${key}.pending.json`,
+ requests,
+ inputDigest: digest,
+ });
+ await expect(prepare(Buffer.from([1]))).rejects.toThrow(
+ "checkpoint was preserved",
+ );
+ expect(await readFile(path, "utf8")).toBe(saved);
+ expect(f.posts).toHaveLength(0);
+ if (kind === "cloned") {
+ expect((await prepare(Buffer.from([2]))).preview.resumed).toBe(false);
+
+ // A migrated approval can be linked to its scoped wire key from an older filename.
+ const linkedPath = join(directory, `${"a".repeat(64)}.pending.json`);
+ await fs.rename(path, linkedPath);
+ const linked = {
+ ...checkpoint,
+ wireKey: key,
+ inputProjection: { ...checkpoint.inputProjection, identityFormat: 2 },
+ };
+ await writeFile(
+ linkedPath,
+ JSON.stringify({ ...linked, legacyApproval: true }),
+ );
+ await expect(prepare(Buffer.from([1]))).rejects.toThrow(
+ "checkpoint was preserved",
+ );
+ // A format-2 request for the plain indexed object is an independent selection.
+ await writeFile(linkedPath, JSON.stringify(linked));
+ expect((await prepare(Buffer.from([1]))).preview.resumed).toBe(false);
+ }
+ },
+);
+
+test("the GitHub reader resumes a released boxed score whose Buffer fit the Cloud request", async () => {
+ const f = await fixture();
+ const row = importedAlert();
+ const extension = {
+ score: new Number(1e-7),
+ bytes: Buffer.from([1]),
+ pad: "",
+ };
+ Object.assign(row.alert, { extension });
+ const normalize = () => normalizeGitHubCodeScanningAlerts([row], "cloud");
+ const prepare = (input = normalize()) =>
+ prepareNormalizedExternalPublication(
+ input,
+ { ...options, provider: "github" },
+ f.deps,
+ );
+
+ // Rebuild the released request without storing the 256 KiB row in the fixture.
+ const priorReader = structuredClone(normalize());
+ priorReader.findings[0]!.evidence.source_data = { ...row };
+ const template = await prepare();
+ const request = JSON.parse(JSON.stringify(template.preview.requests[0]));
+ request.request_id = "5e640fea-52f8-43c2-ba4c-a0822c040458";
+ extension.pad = "x".repeat(260303);
+ const wire = request.items[0].evidence;
+ wire.source_data.alert.extension.bytes = { "0": 1 };
+ wire.source_data.alert.extension.pad = extension.pad;
+ expect(
+ Buffer.byteLength(JSON.stringify(priorReader.findings[0]!.evidence)),
+ ).toBe(256 * 1024);
+ const bodyHash =
+ "69a99d16710c9dc0710268e00b91ecdbb45df094d2918ec9797163332bcd37c5";
+ expect(Buffer.byteLength(JSON.stringify(wire))).toBe(262123);
+ expect(hash("sha256", JSON.stringify(request))).toBe(bodyHash);
+ await expect(prepare()).rejects.toThrow("Cloud limit of 256 KiB");
+ await expect(prepare(priorReader)).rejects.toThrow("Cloud limit of 256 KiB");
+ expect(f.posts).toHaveLength(0);
+
+ await stageReleasedCheckpoint(f, {
+ filename:
+ "c03e748fff72692564d445fc232317c164f16cf16fc5b1969bbee80d98bdb97e.pending.json",
+ requests: [request],
+ inputDigest:
+ "6d0cd82bed91273c26d2d9cd6eea9814f7ce3ac33af58a266f6554617d560362",
+ });
+
+ const resumed = await prepare(priorReader);
+ expect(resumed.preview.resumed).toBe(true);
+ expect(resumed.preview.excluded).toEqual([]);
+ f.state.brokenReadback = true;
+ await expect(resumed.publish()).rejects.toThrow("Readback unavailable");
+ f.state.brokenReadback = false;
+ const completed = await (await prepare()).publish();
+ expect(completed).toMatchObject({
+ status: "complete",
+ ready: 1,
+ excluded: [],
+ unacknowledged: 0,
+ counts: { created: 1 },
+ verified: 1,
+ });
+ expect(completed.receipts.map(({ id }) => id)).toEqual([request.request_id]);
+ expect(f.posts.map((body) => hash("sha256", body))).toEqual([bodyHash]);
+});
diff --git a/sdk/typescript/tests-ts/fixtures/boxed-number-checkpoints.json b/sdk/typescript/tests-ts/fixtures/boxed-number-checkpoints.json
new file mode 100644
index 000000000..78a99106a
--- /dev/null
+++ b/sdk/typescript/tests-ts/fixtures/boxed-number-checkpoints.json
@@ -0,0 +1,104 @@
+{
+ "accountId": "synthetic-account",
+ "request": {
+ "repository": {
+ "id": "repo-example",
+ "repo_connector_id": "github",
+ "environment_id": "environment-example",
+ "reset_marker": "generation-1"
+ },
+ "source": {
+ "provider": "wiz",
+ "source_key": "tenant-example/vulnerability-finding"
+ },
+ "items": [
+ {
+ "client_id": "item-1",
+ "source_finding_id": "boxed-score",
+ "expected_version": 1,
+ "evidence": {
+ "title": "Scanner score",
+ "severity": "high",
+ "source_data": {
+ "score": 2.5
+ },
+ "description": null,
+ "url": null,
+ "advisory_ids": [],
+ "packages": [],
+ "locations": [],
+ "branch": null,
+ "code_revision": null,
+ "source_scan_id": null,
+ "image_digests": [],
+ "source_updated_at": null
+ }
+ }
+ ]
+ },
+ "cases": [
+ {
+ "field": "evidence",
+ "filename": "1b86d2d5fc316cb278171f1a8eabbb4c2662236348fe12d7064a4c2b0b8d76d7.pending.json",
+ "requestId": "e68c368b-9650-4c7a-9560-fab0c24b5a95",
+ "inputProjection": {
+ "inputDigest": "c7582873f1bfaab3cb3a8f5f0b6bdea63e622dfdf3896e4d4c34e3510125082f",
+ "savedEvidenceDigest": "425dc35452a9246322e7751766727cbfb373d5dedce85f143e6c8f1210ce5013"
+ }
+ },
+ {
+ "field": "report",
+ "filename": "7281a4be61bfaf2614d8399d91dbc1a8fd34a2bac0f3e8ae1dfc0fa143fcb9ba.pending.json",
+ "requestId": "5b646989-937e-4497-83ee-6694b967597c",
+ "inputProjection": {
+ "inputDigest": "790edb4daf5954c0d992847a2aa38c665a4e36fe94a2e46cddc62aef4cc83fde",
+ "savedEvidenceDigest": "875a8aec2f1b7c573a6d05d983f6afa0f964a6e1c96d22c8c8691cc2de824ccc"
+ }
+ },
+ {
+ "field": "aliased",
+ "filename": "294b9a84b47d7b08595ae253884111b17b21557f48e90897231da61ab4971035.pending.json",
+ "requestId": "9462e9b9-b78f-49a9-95bd-59bc3d842304",
+ "inputProjection": {
+ "inputDigest": "8f63530e51452c8bb96583dd2bb85705196a209c41258d45be4280e2b5ce424a",
+ "savedEvidenceDigest": "875a8aec2f1b7c573a6d05d983f6afa0f964a6e1c96d22c8c8691cc2de824ccc"
+ }
+ },
+ {
+ "field": "mixed",
+ "filename": "2bdb5e0976da513a32b080995421a304bb498f2ecd8cf1dca323e50e4684cf2c.pending.json",
+ "requestId": "eb070433-a248-4313-a692-21c53effa137",
+ "inputProjection": {
+ "inputDigest": "e0d874045a94c17362500a970e819c07a10d4a27103acabf4f3fd50156050a4d",
+ "savedEvidenceDigest": "425dc35452a9246322e7751766727cbfb373d5dedce85f143e6c8f1210ce5013"
+ }
+ },
+ {
+ "field": "unreported",
+ "filename": "a01f282b3e06ba2cf0ffb43749e47d15620e6f7b057bfb16a4baa3044e90bd6b.pending.json",
+ "requestId": "9ac96055-5fa4-4786-ad5b-a044fb59ce8f",
+ "inputProjection": {
+ "inputDigest": "7fddcc63cf22a90e05f4fb5dc5ef1107661d0a0147ae3eb7e610e840cf663fba",
+ "savedEvidenceDigest": "875a8aec2f1b7c573a6d05d983f6afa0f964a6e1c96d22c8c8691cc2de824ccc"
+ }
+ },
+ {
+ "field": "flag",
+ "filename": "2302ce8a88755173629443d05cb1b296e77e62c99b2c2334116b04677202d96b.pending.json",
+ "requestId": "a47dc563-e876-46dc-b7a9-469dc4c32583",
+ "inputProjection": {
+ "inputDigest": "28f91d7339da91f5497a628f8e084ee938b363d7ce9bd18a0be95115f9769ecf",
+ "savedEvidenceDigest": "c731c2bd0466093f147300606be75bad7b61c6f308fc9fafa8e55c5e2f93049c"
+ }
+ },
+ {
+ "field": "empty-report",
+ "filename": "d14283c9bb988385486811b4a41dcdbf4cf5b6474c5654e087ae0d60c4cd64ad.pending.json",
+ "requestId": "e1c3295f-41a3-4bc7-b6fd-58fab4338933",
+ "inputProjection": {
+ "inputDigest": "ac73456afe6f082161af44bc2134d65c1f75c1dc3ba191ab8a9b7ddf3fd05c73",
+ "savedEvidenceDigest": "425dc35452a9246322e7751766727cbfb373d5dedce85f143e6c8f1210ce5013"
+ }
+ }
+ ]
+}
diff --git a/sdk/typescript/tests-ts/fixtures/boxed-sonar-checkpoint.json b/sdk/typescript/tests-ts/fixtures/boxed-sonar-checkpoint.json
new file mode 100644
index 000000000..7bf69b8c4
--- /dev/null
+++ b/sdk/typescript/tests-ts/fixtures/boxed-sonar-checkpoint.json
@@ -0,0 +1,81 @@
+{
+ "filename": "64800b63401460aadf4376cef6cf4d8aa6c1684d66c0541aa11aa3ce84306799.pending.json",
+ "checkpoint": {
+ "accountId": "synthetic-account",
+ "requests": [
+ {
+ "request_id": "1b018e6f-7f68-40f0-90cc-e855e383f63c",
+ "repository": {
+ "id": "repo-example",
+ "repo_connector_id": "github",
+ "environment_id": "environment-example",
+ "reset_marker": "generation-1"
+ },
+ "source": {
+ "provider": "sonar",
+ "source_key": "tenant-example/vulnerability-finding"
+ },
+ "items": [
+ {
+ "client_id": "item-1",
+ "source_finding_id": "issue-1",
+ "expected_version": 1,
+ "evidence": {
+ "title": "Review input handling",
+ "severity": "high",
+ "source_data": {
+ "format": "sonar-issues/v1",
+ "repositoryUrl": "https://github.com/example/project",
+ "issue": {
+ "key": "issue-1"
+ },
+ "effort": 2.5
+ },
+ "description": null,
+ "url": null,
+ "advisory_ids": [],
+ "packages": [],
+ "locations": [],
+ "branch": null,
+ "code_revision": null,
+ "source_scan_id": null,
+ "image_digests": [],
+ "source_updated_at": null
+ }
+ },
+ {
+ "client_id": "item-2",
+ "source_finding_id": "issue-2",
+ "expected_version": 1,
+ "evidence": {
+ "title": "Review input handling",
+ "severity": "high",
+ "source_data": {
+ "format": "sonar-issues/v1",
+ "repositoryUrl": "https://github.com/example/project",
+ "branch": "main",
+ "issue": {
+ "key": "issue-2"
+ }
+ },
+ "branch": "main",
+ "description": null,
+ "url": null,
+ "advisory_ids": [],
+ "packages": [],
+ "locations": [],
+ "code_revision": null,
+ "source_scan_id": null,
+ "image_digests": [],
+ "source_updated_at": null
+ }
+ }
+ ]
+ }
+ ],
+ "inputProjection": {
+ "inputDigest": "35c7201ca98d5808994aece0d04b7f330ae32f9836d443f9676c329f808dbb58",
+ "savedEvidenceDigest": "5443dab95a2cd679f58b9fcfefc2ccacdc74dad2e6b2697f318883013ab5740a"
+ }
+ }
+}
diff --git a/sdk/typescript/tests-ts/github-dependabot-findings.test.ts b/sdk/typescript/tests-ts/github-dependabot-findings.test.ts
index 03968e221..7a3488117 100644
--- a/sdk/typescript/tests-ts/github-dependabot-findings.test.ts
+++ b/sdk/typescript/tests-ts/github-dependabot-findings.test.ts
@@ -36,6 +36,28 @@ function retainedRepository(
});
}
+test.each(["local", "cloud"] as const)(
+ "validates a boxed score only when its array exports it on %s",
+ (target) => {
+ const alert = githubDependabotAlert();
+ const samples = Object.assign([1], { label: new Number(Infinity) });
+ const accepted = normalizeGitHubDependabotAlerts(
+ [{ ...alert, samples }],
+ target,
+ );
+ expect(accepted.excluded).toEqual([]);
+ expect(
+ JSON.parse(JSON.stringify(accepted.findings[0]!.evidence.source_data))
+ .samples,
+ ).toEqual([1]);
+ const rejected = normalizeGitHubDependabotAlerts(
+ [{ ...alert, samples: [samples.label] }],
+ target,
+ );
+ expect(rejected.excluded[0]!.reason).toContain("finite JSON numbers");
+ },
+);
+
test("matches the affected dependency, retains the original alert, and never invents an installed version or revision", () => {
const alert = githubDependabotAlert();
alert.security_advisory.vulnerabilities.push({
@@ -554,10 +576,17 @@ test("file reader dispatches native lists, selected JSONL and gzip while preserv
`/${repository}/code-scanning/12/`,
);
}
- await writeFile(file, JSON.stringify([importedAlert(), []]));
+ const badAlert = { ...importedAlert(), repository: "example/other" };
+ await writeFile(file, JSON.stringify([importedAlert(), [], badAlert]));
const legacyWithMalformedSibling = await readGitHubFindings(file);
expect(legacyWithMalformedSibling.findings).toHaveLength(1);
- expect(legacyWithMalformedSibling.excluded).toHaveLength(1);
+ expect(legacyWithMalformedSibling.excluded).toHaveLength(2);
+ expect(legacyWithMalformedSibling.excluded[1]).toEqual({
+ position: 3,
+ source_finding_id: badAlert.url,
+ reason:
+ "GitHub alert URL does not match the selected repository and alert.",
+ });
for (const invalid of [
JSON.stringify([alerts[0], importedAlert()]),
'{"number":1',
diff --git a/sdk/typescript/tests-ts/github-findings.test.ts b/sdk/typescript/tests-ts/github-findings.test.ts
index 62d1fe566..fb8239416 100644
--- a/sdk/typescript/tests-ts/github-findings.test.ts
+++ b/sdk/typescript/tests-ts/github-findings.test.ts
@@ -1,8 +1,11 @@
-import { expect, test } from "bun:test";
+import { expect, spyOn, test } from "bun:test";
import { Octokit } from "@octokit/core";
+import { runInNewContext } from "node:vm";
import { importGitHubCodeScanningAlerts } from "../src/github.js";
import { importGitHubAdvisories } from "../src/github-advisories.js";
import { validateExternalEvidence } from "../src/external-import-contract.js";
+import type { ExternalFindings } from "../src/external-findings.js";
+import { previousReaderSelection } from "../src/reader-selection.js";
import {
normalizeGitHubCodeScanningAlerts,
normalizeGitHubAdvisories,
@@ -14,6 +17,590 @@ import {
importedAdvisory,
} from "./support/github-import-fixtures.js";
+function priorSelection(result: ExternalFindings) {
+ return previousReaderSelection(result, structuredClone(result));
+}
+
+function alertWith(fields: Record<string, unknown>, ref?: string) {
+ const row = importedAlert(ref);
+ Object.assign(row.alert, fields);
+ return row;
+}
+
+function sourceAlert(result: ExternalFindings | undefined, index = 0) {
+ return result?.findings[index]?.evidence.source_data?.["alert"] as
+ Record<string, unknown> | undefined;
+}
+
+test("captures a record-root serializer with its selection index once", () => {
+ const row = importedAlert(),
+ keys: string[] = [];
+ const instance = row.alert["most_recent_instance"] as Record<string, unknown>;
+ Object.setPrototypeOf(row, {
+ toJSON(key: string) {
+ keys.push(key);
+ return {
+ ...row,
+ alert: {
+ ...row.alert,
+ most_recent_instance: { ...instance, ref: "refs/heads/" + key },
+ },
+ };
+ },
+ });
+ const output = normalizeGitHubCodeScanningAlerts([row]);
+ expect(output.findings[0]?.evidence.branch).toBe("refs/heads/0");
+ expect(keys).toEqual(["0"]);
+ const old = priorSelection(output);
+ expect(old?.sourceFindingIds).toHaveLength(1);
+ expect(old?.snapshot).toBeUndefined();
+});
+
+test.each(["object", "class"])(
+ "flags hidden %s rule fields during replay",
+ (kind) => {
+ class Rule {}
+ const row = importedAlert();
+ const rule = row.alert["rule"] as Record<string, unknown>;
+ if (kind === "class") Object.setPrototypeOf(rule, Rule.prototype);
+ Object.defineProperty(rule, "name", {
+ enumerable: false,
+ value: "Hidden title",
+ });
+ Object.defineProperty(rule, "security_severity_level", {
+ enumerable: false,
+ value: "critical",
+ });
+ const current = normalizeGitHubCodeScanningAlerts([row]);
+ const id = current.findings[0]!.source_finding_id;
+ expect(current.findings[0]?.evidence.title).not.toBe("Hidden title");
+ expect(current.findings[0]?.evidence.severity).toBe("unknown");
+ const prior = priorSelection(current);
+ expect(prior?.snapshot).toBeUndefined();
+ expect(prior?.uncertainFindingIds).toEqual([id]);
+ },
+);
+
+test("advisory membership with a hidden numeric getter is uncertain without reading it", () => {
+ const first = importedAdvisory("repository"),
+ second = importedAdvisory("global");
+ let reads = 0;
+ const score = Object.defineProperty(new Number(7), "bad", {
+ enumerable: true,
+ get: () => {
+ reads++;
+ return Infinity;
+ },
+ });
+ Object.assign(second.advisory, { score });
+ const current = normalizeGitHubAdvisories([first, second]);
+ expect(reads).toBe(0);
+ expect(current.findings).toHaveLength(2);
+ const prior = priorSelection(current);
+ expect(prior?.snapshot).toBeUndefined();
+ expect(prior?.uncertainFindingIds).toEqual([
+ current.findings[1]!.source_finding_id,
+ ]);
+ expect(prior?.knownFindings?.map((f) => f.source_finding_id)).toEqual([
+ current.findings[0]!.source_finding_id,
+ ]);
+});
+
+test("later occurrence IDs can depend on the key passed to an earlier GitHub serializer", () => {
+ function alert(number: number) {
+ const row = importedAlert();
+ row.number = number;
+ row.url = row.url.replace(/12$/, String(number));
+ Object.assign(row.alert, { number, html_url: row.url });
+ return row;
+ }
+ const stable = alert(11),
+ first = alert(12),
+ later = alert(13);
+ const instance = later.alert["most_recent_instance"] as Record<
+ string,
+ unknown
+ >;
+ Object.defineProperty(first, "toJSON", {
+ value(key: string) {
+ instance["ref"] = "refs/heads/" + key;
+ return { ...first };
+ },
+ });
+ const current = normalizeGitHubCodeScanningAlerts([stable, first, later]);
+ expect(current.findings[2]?.evidence.branch).toBe("refs/heads/1");
+ const prior = priorSelection(current);
+ expect(prior?.knownFindings?.map((row) => row.source_finding_id)).toEqual([
+ current.findings[0]!.source_finding_id,
+ ]);
+ expect(prior?.uncertainFindingIds).toContain(
+ current.findings[2]!.source_finding_id,
+ );
+ expect(prior?.unprovenOccurrenceCount).toBe(1);
+});
+
+test("a getter can mutate a later GitHub row before its historical evidence is observed", () => {
+ const first = importedAlert("refs/heads/first");
+ const middle = importedAlert("refs/heads/middle");
+ const last = importedAlert("refs/heads/last");
+ const counter = { count: 0 };
+ Object.assign(middle.alert, {
+ extension: {
+ get sample() {
+ return ++counter.count;
+ },
+ },
+ });
+ Object.assign(last.alert, { extension: counter });
+ const fresh = normalizeGitHubCodeScanningAlerts([first, middle, last]);
+ const prior = priorSelection(fresh);
+ expect(counter.count).toBe(1);
+ expect(prior?.sourceFindingIds).toEqual(
+ fresh.findings.map((row) => row.source_finding_id),
+ );
+ expect(prior?.uncertainFindingIds).toEqual([
+ fresh.findings[1]!.source_finding_id,
+ ]);
+ expect(prior?.knownFindings?.map((row) => row.source_finding_id)).toEqual([
+ fresh.findings[0]!.source_finding_id,
+ ]);
+});
+
+test.each(["malformed details", "hidden ID", "unobservable ID"] as const)(
+ "retains the old scope when an advisory has %s",
+ (kind) => {
+ const first = importedAdvisory("repository"),
+ second = importedAdvisory("global");
+ const oldId = normalizeGitHubAdvisories([second]).findings[0]!
+ .source_finding_id;
+ if (kind === "hidden ID")
+ Object.defineProperty(second, "ghsaId", {
+ value: second.ghsaId,
+ enumerable: false,
+ });
+ else {
+ const old = second.ghsaId;
+ if (kind === "unobservable ID")
+ Object.defineProperty(second, "ghsaId", {
+ get: () => old,
+ enumerable: true,
+ });
+ Object.defineProperty(second.advisory, "score", {
+ enumerable: true,
+ get: () => 7,
+ });
+ Object.defineProperty(second.advisory, "toJSON", {
+ value: () => ({ ...second.advisory, vulnerabilities: "invalid" }),
+ });
+ }
+ const fresh = normalizeGitHubAdvisories([first, second]);
+ expect(fresh.findings).toHaveLength(1);
+ const hints = priorSelection(fresh);
+ expect(hints?.snapshot).toBeUndefined();
+ if (kind === "unobservable ID")
+ expect(hints?.unprovenOccurrenceCount).toBe(1);
+ else expect(hints?.uncertainFindingIds).toContain(oldId);
+ },
+);
+
+test("an inert toJSON field retains ordinary advisory selection semantics", () => {
+ const item = Object.assign(importedAdvisory(), { toJSON: null });
+ const result = normalizeGitHubAdvisories([item]);
+ expect(result.findings).toHaveLength(1);
+ expect(priorSelection(result)).toBeUndefined();
+});
+
+test("a caller-added report accompanies the earlier selection without changing its row binding", () => {
+ const row = alertWith({ bytes: Buffer.from([5, 17, 129]) });
+ const result = normalizeGitHubCodeScanningAlerts([row]);
+ const input = { ...result, report: { run: "1" } };
+ const exported = structuredClone(input);
+ const prior = previousReaderSelection(input, exported)?.snapshot;
+ expect(prior?.report).toEqual({ run: "1" });
+ expect(sourceAlert(prior)).toMatchObject({
+ bytes: Uint8Array.from([5, 17, 129]),
+ });
+ input.report.run = "2";
+ expect(prior?.report).toEqual({ run: "1" });
+ expect(priorSelection(input)?.snapshot?.report).toEqual({ run: "2" });
+ expect(priorSelection(result)?.snapshot?.report).toBeUndefined();
+ expect(
+ previousReaderSelection(input, exported, null)?.snapshot,
+ ).toBeUndefined();
+ expect(
+ previousReaderSelection(input, { ...exported, read: 2 })?.snapshot,
+ ).toBeUndefined();
+});
+
+test("reader provenance distinguishes native mutations hidden by unchanged JSON", () => {
+ const item = alertWith({
+ score: Object.assign(new Number(7), { toJSON: () => 9 }),
+ });
+ const result = normalizeGitHubCodeScanningAlerts([item]);
+ const exported = structuredClone(result);
+ expect(
+ previousReaderSelection({ ...result }, exported)?.snapshot?.findings,
+ ).toHaveLength(1);
+ const alert = sourceAlert(result)!;
+ alert["score"] = Object.assign(new Number(8), { toJSON: () => 9 });
+ expect(JSON.parse(JSON.stringify(result))).toEqual(exported);
+ expect(previousReaderSelection(result, exported)?.snapshot).toBeUndefined();
+});
+
+test("locally normalized GitHub output retains the original Buffer for Cloud checkpoint verification", () => {
+ const row = alertWith({ payload: Buffer.from([5, 17, 129]) });
+ const current = normalizeGitHubCodeScanningAlerts([row], "local");
+ expect(sourceAlert(current)).toMatchObject({
+ payload: { type: "Buffer", data: [5, 17, 129] },
+ });
+ const prior = priorSelection(current);
+ expect(sourceAlert(prior?.snapshot)).toMatchObject({
+ payload: Uint8Array.from([5, 17, 129]),
+ });
+});
+
+test("an oversized local export retains the small native GitHub selection", () => {
+ class Metric {
+ value = 7;
+ toJSON() {
+ return "x".repeat(300 * 1024);
+ }
+ }
+ const row = alertWith({ metric: new Metric() });
+ const current = normalizeGitHubCodeScanningAlerts([row], "local");
+ const evidence = current.findings[0]!.evidence;
+ expect(() => validateExternalEvidence(evidence, "cloud")).toThrow("256 KiB");
+ const prior = priorSelection(current);
+ expect(sourceAlert(prior?.snapshot)).toMatchObject({
+ metric: { value: 7 },
+ });
+});
+
+test("local reader provenance serializes selections one finding at a time", () => {
+ const first = alertWith({ payload: Buffer.from([5, 17]) });
+ const stringify = spyOn(JSON, "stringify");
+ try {
+ const current = normalizeGitHubCodeScanningAlerts(
+ [first, importedAlert("refs/heads/second")],
+ "local",
+ );
+ const lookup = structuredClone(current);
+ expect(
+ previousReaderSelection(current, lookup)?.snapshot?.findings,
+ ).toHaveLength(2);
+ const batches = [current.findings, lookup.findings];
+ const serializedBatch = stringify.mock.calls.some(
+ ([value]) => batches.includes(value) || batches.includes(value?.findings),
+ );
+ expect(serializedBatch).toBe(false);
+ } finally {
+ stringify.mockRestore();
+ }
+});
+
+test("a deeply shared graph excluded from historical evidence is not copied again for comparison", () => {
+ let graph: unknown = {};
+ for (let level = 0; level < 21; level++)
+ graph = { left: graph, right: graph };
+ class Extension {
+ graph = graph;
+ toJSON() {
+ return "summary";
+ }
+ }
+ const row = alertWith({ extension: new Extension() });
+ const clone = spyOn(globalThis, "structuredClone");
+ try {
+ const output = normalizeGitHubCodeScanningAlerts([row]);
+ expect(sourceAlert(output)).toMatchObject({
+ extension: "summary",
+ });
+ expect(priorSelection(output)?.snapshot?.excluded[0]?.reason).toContain(
+ "depth",
+ );
+ expect(clone.mock.calls.filter(([value]) => value === row)).toHaveLength(1);
+ } finally {
+ clone.mockRestore();
+ }
+});
+
+test.each([7, 2 ** 53])(
+ "a compact export with a deeply shared raw number %i uses the emitted value",
+ (value) => {
+ let reads = 0;
+ let graph: unknown = {
+ score: value,
+ get unused() {
+ reads++;
+ return 1;
+ },
+ };
+ for (let i = 0; i < 28; i++) graph = { left: graph, right: graph };
+ const row = alertWith({ extension: { graph, toJSON: () => "summary" } });
+ const output = normalizeGitHubCodeScanningAlerts([row]);
+ expect(reads).toBe(0);
+ expect(sourceAlert(output)).toMatchObject({
+ extension: "summary",
+ });
+ },
+);
+
+test.each(["local", "cloud"] as const)(
+ "observes a stateful mapped GitHub ref only once on %s",
+ (target) => {
+ function fixture() {
+ const row = importedAlert();
+ let count = 0;
+ Object.defineProperty(row.alert["most_recent_instance"], "ref", {
+ enumerable: true,
+ get: () => "refs/heads/r" + ++count,
+ });
+ return { row, count: () => count };
+ }
+ const direct = fixture(),
+ saved = fixture();
+ const result = normalizeGitHubCodeScanningAlerts([direct.row], target);
+ const exported = normalizeGitHubCodeScanningAlerts(
+ JSON.parse(JSON.stringify([saved.row])),
+ target,
+ );
+ expect(result).toEqual(exported);
+ expect(result.findings[0]?.evidence.branch).toBe("refs/heads/r1");
+ expect(direct.count()).toBe(1);
+ },
+);
+
+test.each(["local", "cloud"] as const)(
+ "maps the exported GitHub occurrence and checks its identity on %s",
+ (target) => {
+ const rows = ["main", "review"].map((ref) => {
+ const row = importedAlert("refs/heads/" + ref);
+ const instance = row.alert["most_recent_instance"] as Record<
+ string,
+ unknown
+ >;
+ Object.setPrototypeOf(instance, {
+ toJSON(this: typeof instance) {
+ return { ...this, ref: "refs/heads/exported" };
+ },
+ });
+ return row;
+ });
+ const plain = JSON.parse(JSON.stringify(rows));
+ const read = normalizeGitHubCodeScanningAlerts([rows[0]!], target);
+ expect(read).toEqual(normalizeGitHubCodeScanningAlerts([plain[0]], target));
+ expect(read.findings[0]!.evidence.branch).toBe("refs/heads/exported");
+ expect(() => normalizeGitHubCodeScanningAlerts(rows, target)).toThrow(
+ "Duplicate source finding id",
+ );
+ if (target === "cloud") {
+ const old = priorSelection(read)?.snapshot;
+ expect(old?.findings[0]!.evidence.branch).toBe("refs/heads/main");
+ expect(old?.findings[0]!.source_finding_id).not.toBe(
+ read.findings[0]!.source_finding_id,
+ );
+ }
+ },
+);
+
+test.each(["local", "cloud"] as const)(
+ "retains exactly the exported GitHub values on %s",
+ (target) => {
+ const score = Object.assign(new Number(7), {
+ toJSON: (key: string) => (key === "score" ? 9 : 1e-7),
+ });
+ class Samples extends Array<number> {
+ toJSON() {
+ return { count: this.length };
+ }
+ }
+ const extension = {
+ score,
+ get alias() {
+ return score;
+ },
+ samples: new Samples(1, 2),
+ warning: Object.assign(new Error("scanner warning"), { code: "W1" }),
+ bytes: Buffer.from([1, 2]),
+ };
+ const row = alertWith({ extension });
+ const read = normalizeGitHubCodeScanningAlerts([row], target);
+ expect(read.excluded).toEqual([]);
+ const alert = sourceAlert(read)!;
+ expect(alert["extension"]).toEqual({
+ score: 9,
+ alias: 1e-7,
+ samples: { count: 2 },
+ warning: { code: "W1" },
+ bytes: { type: "Buffer", data: [1, 2] },
+ });
+ extension.warning.code = "later";
+ expect(alert["extension"]).toMatchObject({ warning: { code: "W1" } });
+ },
+);
+
+test.each(["local", "cloud"] as const)(
+ "excludes raw GitHub numbers that lose precision on %s",
+ (target) => {
+ const raw = (JSON as JSON & { rawJSON(token: string): unknown }).rawJSON;
+ const rounded = alertWith(
+ { metric: raw("1.0000000000000001") },
+ "refs/heads/rounded",
+ );
+ const exact = alertWith({ metric: raw("10.00E-8"), count: raw("1.00") });
+
+ const result = normalizeGitHubCodeScanningAlerts([rounded, exact], target);
+ expect(result.excluded).toMatchObject([
+ { position: 1, reason: expect.stringContaining("loses precision") },
+ ]);
+ expect(result.findings).toHaveLength(1);
+ expect(sourceAlert(result)).toMatchObject({
+ metric: 1e-7,
+ count: 1,
+ });
+ },
+);
+
+test.each(["local", "cloud"] as const)(
+ "checks emitted GitHub numbers and retains previous numeric exclusions on %s",
+ (target) => {
+ function reason(value: unknown) {
+ const row = alertWith({ metric: value });
+ return normalizeGitHubCodeScanningAlerts([row], target).excluded[0]
+ ?.reason;
+ }
+ for (const value of [
+ 2 ** 53,
+ runInNewContext("new Number(2 ** 53)"),
+ Object.assign(new Number(7), { toJSON: () => 2 ** 53 }),
+ ])
+ expect(reason(value)).toContain("safe integer");
+ expect(reason({ toJSON: () => NaN })).toContain("finite JSON");
+ for (const [metric, exported, previousReason] of [
+ [Object.assign(new Number(7), { hidden: Infinity }), 7, "finite JSON"],
+ [{ x: 2 ** 53, toJSON: () => "summary" }, "summary", "safe integer"],
+ ] as const) {
+ const allowed = importedAlert("refs/heads/first");
+ const row = alertWith({ metric });
+ const result = normalizeGitHubCodeScanningAlerts([allowed, row], target);
+ expect(result.excluded).toEqual([]);
+ expect(sourceAlert(result, 1)).toMatchObject({ metric: exported });
+ if (target === "cloud") {
+ const previous = priorSelection(result)?.snapshot;
+ expect(previous?.findings.map((f) => f.source_finding_id)).toEqual([
+ result.findings[0]!.source_finding_id,
+ ]);
+ expect(previous?.excluded[0]?.reason).toContain(previousReason);
+ }
+ }
+ },
+);
+
+test.each(["cloud", "local"] as const)(
+ "accepts a boxed integer explicitly exported as safe JSON on %s",
+ (target) => {
+ for (const emitted of ["9007199254740992", 7]) {
+ const row = alertWith({
+ metric: Object.assign(new Number(2 ** 53), { toJSON: () => emitted }),
+ });
+ const normalized = normalizeGitHubCodeScanningAlerts([row], target);
+ expect(normalized).toEqual(
+ normalizeGitHubCodeScanningAlerts(
+ JSON.parse(JSON.stringify([row])),
+ target,
+ ),
+ );
+ expect(sourceAlert(normalized)).toMatchObject({ metric: emitted });
+ }
+ },
+);
+
+test("a data-field number's original native value survives only for verifying an earlier approval", () => {
+ const score = Object.assign(new Number(7), { toJSON: () => 9 });
+ const row = alertWith({ score });
+ const result = normalizeGitHubCodeScanningAlerts([row]);
+ const previous = priorSelection(result);
+ const saved = sourceAlert(previous?.snapshot)!;
+ expect(
+ previousReaderSelection({ ...result }, structuredClone(result))?.snapshot,
+ ).toBe(previous?.snapshot);
+ expect(
+ previousReaderSelection({ ...result }, { ...result, read: result.read + 1 })
+ ?.snapshot,
+ ).toBeUndefined();
+ expect(Number.prototype.valueOf.call(saved["score"])).toBe(7);
+ expect(sourceAlert(result)).toMatchObject({
+ score: 9,
+ });
+});
+
+test("uses the actual first wire and preserves only an occurrence hint for getter-dependent approvals", () => {
+ const score = Object.assign(new Number(7), { toJSON: () => 9 });
+ let reads = 0;
+ const shared = {
+ score,
+ get sample() {
+ return ++reads;
+ },
+ };
+ const rows = ["main", "review", "excluded"].map((name) =>
+ alertWith({ shared }, "refs/heads/" + name),
+ );
+ Object.assign(rows[2]!.alert, {
+ extra: { toJSON: () => "retained" },
+ pad: "x".repeat(256 * 1024),
+ });
+ const output = normalizeGitHubCodeScanningAlerts(rows);
+ const fresh = output.findings.map(
+ ({ evidence }) =>
+ (
+ evidence.source_data!["alert"] as {
+ shared: { score: number; sample: number };
+ }
+ ).shared,
+ );
+ expect(fresh).toEqual([
+ { score: 9, sample: 1 },
+ { score: 9, sample: 2 },
+ ]);
+ expect(reads).toBe(3);
+ const prior = priorSelection(output);
+ expect(prior?.snapshot).toBeUndefined();
+ expect(prior?.unprovenIdPrefixes).toEqual([
+ "github.com/example/project/code-scanning/12/",
+ ]);
+});
+
+test("an exported row over the Cloud byte limit can have an exact historical selection", () => {
+ const raw = { score: new Number(1e-7), bytes: Buffer.from([1]), pad: "" };
+ const row = alertWith({ raw });
+ const read = () => normalizeGitHubCodeScanningAlerts([row]);
+ const size = Buffer.byteLength(JSON.stringify(read().findings[0]!.evidence));
+ raw.pad = "x".repeat(256 * 1024 - size - 1);
+ expect(read().findings).toHaveLength(1);
+ raw.pad += "x";
+ const current = read();
+ expect(current.findings).toHaveLength(0);
+ expect(current.excluded[0]?.reason).toContain("256 KiB");
+ const old = priorSelection(current)?.snapshot;
+ expect(old?.findings).toHaveLength(1);
+ expect(old?.excluded).toEqual([]);
+});
+
+test("applies the Cloud byte limit to numbers returned by a vendor serializer", () => {
+ const score = Object.assign(new Number(1), { toJSON: () => 123456 });
+ const extension = { pad: "", formatted: { toJSON: () => ({ score }) } };
+ const row = alertWith({ extension });
+ const read = () => normalizeGitHubCodeScanningAlerts([row]);
+ const evidence = read().findings[0]!.evidence;
+ extension.pad = "x".repeat(
+ 256 * 1024 - Buffer.byteLength(JSON.stringify(evidence)),
+ );
+ expect(read().findings).toHaveLength(1);
+ extension.pad += "x";
+ expect(read().excluded[0]?.reason).toContain("256 KiB");
+});
+
test("code-scanning mapping preserves full evidence and selected instance status without assessing it", () => {
const imported = importedAlert("refs/heads/review");
const result = normalizeGitHubCodeScanningAlerts([imported]);
@@ -68,8 +655,10 @@ test("SARIF levels do not fabricate security severity and malformed siblings rem
delete noRating.alert.rule.security_severity_level;
const badRepository = importedAlert("refs/heads/other");
badRepository.repository = "example/wrong";
- const result = normalizeGitHubCodeScanningAlerts([badRepository, noRating]);
- expect(result.read).toBe(2);
+ const rows = [badRepository, noRating];
+ rows.length = 3;
+ const result = normalizeGitHubCodeScanningAlerts(rows);
+ expect(result.read).toBe(3);
expect(result.findings).toHaveLength(1);
expect(result.findings[0]!.evidence.severity).toBe("unknown");
expect(result.excluded).toEqual([
@@ -79,6 +668,11 @@ test("SARIF levels do not fabricate security severity and malformed siblings rem
reason:
"GitHub alert URL does not match the selected repository and alert.",
},
+ {
+ position: 3,
+ source_finding_id: null,
+ reason: "GitHub finding must be an object.",
+ },
]);
});
@@ -343,3 +937,124 @@ test("real GitHub readers and mappings preserve explicit ports in repository evi
expect(codeIds.size).toBe(3);
expect(advisoryIds.size).toBe(3);
});
+
+test("Cloud rejects over-depth exported GitHub rows before reading shared leaves", () => {
+ let reads = 0;
+ let graph: unknown = {
+ get value() {
+ reads++;
+ return 1;
+ },
+ };
+ for (let i = 0; i < 7; i++) graph = { left: graph, right: graph };
+ for (let i = 0; i < 22; i++) graph = { next: graph };
+ const row = alertWith({ extension: graph });
+
+ const output = normalizeGitHubCodeScanningAlerts([row]);
+ expect(output.excluded[0]?.reason).toBe(
+ "Evidence exceeds the Cloud JSON depth limit of 20.",
+ );
+ expect(reads).toBe(0);
+ expect(
+ normalizeGitHubCodeScanningAlerts([row], "local").findings,
+ ).toHaveLength(1);
+ expect(reads).toBe(128);
+});
+
+test("an over-depth exported GitHub Buffer can retain its valid native selection", () => {
+ let graph: unknown = Buffer.from([7]);
+ for (let i = 0; i < 16; i++) graph = { next: graph };
+ const row = alertWith({ extension: graph });
+
+ const previous = normalizeGitHubCodeScanningAlerts([structuredClone(row)]);
+ const current = normalizeGitHubCodeScanningAlerts([row]);
+ expect(current.excluded[0]?.reason).toBe(
+ "Evidence exceeds the Cloud JSON depth limit of 20.",
+ );
+ expect(previous.findings).toHaveLength(1);
+ expect(
+ JSON.stringify(previousReaderSelection(current, current)?.snapshot),
+ ).toBe(JSON.stringify(previous));
+});
+
+test("an over-depth GitHub export cannot prove a custom serializer's old byte limit", () => {
+ for (const oldPayload of [{ tiny: true }, { pad: "x".repeat(256 * 1024) }]) {
+ const row = importedAlert();
+ let graph: unknown = 1;
+ for (let i = 0; i < 24; i++) graph = { next: graph };
+ const keys: string[] = [];
+ Object.defineProperty(row, "toJSON", {
+ value(key: string) {
+ keys.push(key);
+ return key === "source_data" ? oldPayload : { graph };
+ },
+ });
+ const current = normalizeGitHubCodeScanningAlerts([row]);
+ expect(keys).toEqual(["0"]);
+ expect(current.excluded[0]?.reason).toContain("depth limit");
+ expect(previousReaderSelection(current, current)?.snapshot).toBeUndefined();
+ expect(
+ previousReaderSelection(current, current)?.sourceFindingIds,
+ ).toHaveLength(1);
+ }
+});
+
+test("discarded native Buffer fields cannot establish an old GitHub selection", () => {
+ const buffer = Buffer.from([7]);
+ let extra: unknown = 7,
+ graph: unknown = buffer;
+ for (let i = 0; i < 24; i++) extra = { next: extra };
+ Object.assign(buffer, { extra });
+ for (let i = 0; i < 16; i++) graph = { next: graph };
+ const row = alertWith({ extension: graph });
+ const current = normalizeGitHubCodeScanningAlerts([row]);
+ expect(current.excluded[0]?.reason).toContain("depth limit");
+ expect(previousReaderSelection(current, current)?.snapshot).toBeUndefined();
+});
+
+test("historical GitHub mapping cannot use a native clone with a hidden array index", () => {
+ const row = importedAlert();
+ const tags = Object.defineProperty([], "0", {
+ value: "external/cwe/cwe-089",
+ });
+ Object.assign(row.alert.rule!, { tags });
+ const result = normalizeGitHubCodeScanningAlerts([row]);
+ expect(result.findings[0]?.evidence.details?.weakness_ids).toEqual([
+ "CWE-89",
+ ]);
+ const previous = priorSelection(result);
+ expect(previous?.snapshot).toBeUndefined();
+ expect(previous?.uncertainFindingIds).toEqual([
+ result.findings[0]!.source_finding_id,
+ ]);
+});
+
+test("a failed GitHub serializer retains an own URL without reading a URL getter", () => {
+ const row = importedAlert();
+ let calls = 0;
+ Object.assign(row, {
+ toJSON() {
+ calls++;
+ throw new Error("synthetic export failure");
+ },
+ });
+ expect(normalizeGitHubCodeScanningAlerts([row]).excluded).toEqual([
+ {
+ position: 1,
+ source_finding_id: row.url,
+ reason: "synthetic export failure",
+ },
+ ]);
+ let urlReads = 0;
+ Object.defineProperty(row, "url", {
+ get() {
+ urlReads++;
+ return "unused";
+ },
+ });
+ expect(
+ normalizeGitHubCodeScanningAlerts([row]).excluded[0]?.source_finding_id,
+ ).toBeNull();
+ expect(urlReads).toBe(0);
+ expect(calls).toBe(2);
+});
diff --git a/sdk/typescript/tests-ts/inspector-findings.test.ts b/sdk/typescript/tests-ts/inspector-findings.test.ts
index da5716aae..a659ac2c5 100644
--- a/sdk/typescript/tests-ts/inspector-findings.test.ts
+++ b/sdk/typescript/tests-ts/inspector-findings.test.ts
@@ -1,3 +1,4 @@
+import { previousReaderSelection } from "../src/reader-selection.js";
import { expect, test } from "bun:test";
import { mkdtemp, rm, writeFile } from "node:fs/promises";
import { tmpdir } from "node:os";
@@ -617,6 +618,64 @@ test("only surviving rows share project proof, independently of input order", ()
}
});
+function scopedProjectRows() {
+ const witness = projectRow("a", enterprise, "GITHUB");
+ const dependent = projectRow("b");
+ const independent = projectRow("c", enterprise, "GITHUB");
+ Object.assign(independent.resources[0]!.details.codeRepository, {
+ integrationArn: undefined,
+ });
+ const rows = [witness, dependent, independent];
+ for (const row of rows) row.resources[0]!.id = "example/project";
+ return { rows, witness, dependent, independent };
+}
+
+test.each([
+ [0.00001, 262_145, false],
+ [1e-7, 262_144, true],
+] as const)(
+ "a size change rechecks provider proof for each selected subset (%d)",
+ (score, bytes, previouslyAccepted) => {
+ const { rows, witness, independent } = scopedProjectRows();
+ Object.assign(witness, { score: new Number(score), pad: "" });
+ const evidence = normalize(rows, enterprise).findings[0]!.evidence;
+ Object.assign(witness, {
+ pad: "p".repeat(bytes - Buffer.byteLength(JSON.stringify(evidence))),
+ });
+ const current = normalize(rows, enterprise);
+ const previous = previousReaderSelection(
+ current,
+ JSON.parse(JSON.stringify(current)),
+ );
+ const all = rows.map((row) => row.findingArn);
+ const onlyIndependent = [independent.findingArn];
+ expect(current.findings.map((row) => row.source_finding_id)).toEqual(
+ previouslyAccepted ? onlyIndependent : all,
+ );
+ expect(
+ previous?.snapshot?.findings.map((row) => row.source_finding_id),
+ ).toEqual(previouslyAccepted ? all : onlyIndependent);
+ },
+);
+
+test("an uncertain Inspector row can obscure the membership of its dependent", () => {
+ const { rows, witness, dependent, independent } = scopedProjectRows();
+ Object.assign(witness, { score: new Number(NaN) });
+ const current = normalize(rows, enterprise);
+ const prior = previousReaderSelection(
+ current,
+ JSON.parse(JSON.stringify(current)),
+ );
+ expect(prior?.uncertainFindingIds).toEqual([
+ witness.findingArn,
+ dependent.findingArn,
+ ]);
+ expect(prior?.knownFindings?.map((row) => row.source_finding_id)).toEqual([
+ independent.findingArn,
+ ]);
+ expect(prior?.snapshot).toBeUndefined();
+});
+
test("after the only provider hint is excluded, valid local rows still share their native URL", () => {
const source = "https://scm.example.test/Owner/Repo";
const duplicate = projectRow("a", "Owner/Repo", "GITHUB");
diff --git a/sdk/typescript/tests-ts/opengrep-findings.test.ts b/sdk/typescript/tests-ts/opengrep-findings.test.ts
index dd94e24e4..5429f75ea 100644
--- a/sdk/typescript/tests-ts/opengrep-findings.test.ts
+++ b/sdk/typescript/tests-ts/opengrep-findings.test.ts
@@ -7,6 +7,7 @@ import {
normalizeOpenGrepFindings as normalize,
readOpenGrepFindings,
} from "../src/opengrep-findings.js";
+import { previousReaderSelection } from "../src/reader-selection.js";
import {
opengrepJson,
opengrepTraceJson,
@@ -174,6 +175,80 @@ test("grouped observations still honor the existing Cloud evidence size contract
);
});
+test.each(["group", "trace"] as const)(
+ "a size change affecting a %s keeps the unrelated occurrence proven",
+ (changed) => {
+ const input = opengrepTraceJson();
+ const second = input.results[1]!;
+ if (changed === "trace") second.extra.message = "x".repeat(140_000);
+ // The released reader counted a boxed decimal by its JSON spelling.
+ Object.assign(second.extra.metadata, {
+ score: new Number(0.00001),
+ pad: "",
+ });
+ const group = normalize(input, options).findings[0]!;
+ const bytes = Buffer.byteLength(JSON.stringify(group.evidence));
+ Object.assign(second.extra.metadata, { pad: "p".repeat(262_145 - bytes) });
+ const unrelated = structuredClone(input.results[0]!);
+ unrelated.check_id = "example.unrelated";
+ input.results.push(unrelated);
+ const current = normalize(input, options);
+ expect(current.excluded).toEqual([]);
+ expect(current.findings).toHaveLength(2);
+ const known = current.findings.filter(
+ (row) => row.source_finding_id !== group.source_finding_id,
+ );
+ expect(known).toHaveLength(1);
+ const previous = previousReaderSelection(
+ current,
+ JSON.parse(JSON.stringify(current)),
+ );
+ expect(previous?.uncertainFindingIds).toEqual([group.source_finding_id]);
+ expect(previous?.knownFindings).toEqual(known);
+ const local = normalize(input, options, "local");
+ expect(
+ previousReaderSelection(local, JSON.parse(JSON.stringify(local))),
+ ).toBeUndefined();
+ },
+);
+
+test.each([
+ { score: 0.00001, bytes: 262_145, freshCount: 2, priorCount: 1 },
+ { score: 1e-7, bytes: 262_144, freshCount: 1, priorCount: 2 },
+])(
+ "retains exact singleton membership at $bytes bytes",
+ ({ score, bytes, freshCount, priorCount }) => {
+ const input = opengrepJson();
+ const stable = normalize(input, options).findings[0]!;
+ const candidate = {
+ ...input.results[0]!,
+ check_id: "example.boundary",
+ score: new Number(score),
+ padding: "",
+ };
+ const selected = normalize({ ...input, results: [candidate] }, options)
+ .findings[0]!;
+ candidate.padding = "x".repeat(
+ bytes - Buffer.byteLength(JSON.stringify(selected.evidence)),
+ );
+ input.results.push(candidate);
+ const current = normalize(input, options);
+ expect(current.findings).toHaveLength(freshCount);
+ const prior = previousReaderSelection(
+ current,
+ JSON.parse(JSON.stringify(current)),
+ );
+ const expectedIds =
+ priorCount === 1
+ ? [stable.source_finding_id]
+ : [stable.source_finding_id, selected.source_finding_id].sort();
+ expect(
+ prior?.snapshot?.findings.map((row) => row.source_finding_id),
+ ).toEqual(expectedIds);
+ expect(prior?.uncertainFindingIds).toBeUndefined();
+ },
+);
+
test("an oversized trace leaves the publishable traces at its source occurrence", () => {
const input = opengrepJson();
const valid = input.results[0]!;
diff --git a/sdk/typescript/tests-ts/wiz-findings.test.ts b/sdk/typescript/tests-ts/wiz-findings.test.ts
index f8105bba2..59952b182 100644
--- a/sdk/typescript/tests-ts/wiz-findings.test.ts
+++ b/sdk/typescript/tests-ts/wiz-findings.test.ts
@@ -18,6 +18,7 @@ import { main } from "../src/cli.js";
import {
ExternalPublicationError,
prepareExternalPublication,
+ prepareNormalizedExternalPublication,
} from "../src/external-findings-publish.js";
import { DEFAULT_CLOUD_BASE_URL } from "../src/cloud-endpoint.js";
import { validateExternalEvidence } from "../src/external-import-contract.js";
@@ -1614,6 +1615,51 @@ function legacyCheckpointKey(identity: unknown[]) {
return hash("sha256", encoded);
}
+test("a normalized Wiz retry requires a known locale to reuse its acknowledged receipt", async () => {
+ const f = await previewCheckpointFixture();
+ const input = await readVendorFindings(f.file);
+ const data = input.findings[0]!.evidence.source_data!;
+ Object.defineProperty(data, "toJSON", { value: () => ({ ...data, Z: 7 }) });
+ const prepare = (selected = input) =>
+ prepareNormalizedExternalPublication(selected, f.options, {
+ ...f.deps,
+ environment: {
+ ...f.environment,
+ CODEX_SECURITY_CLOUD_BASE_URL: DEFAULT_CLOUD_BASE_URL,
+ },
+ });
+ const early = await prepare();
+ const foreign = join(f.directory, hostedForeignName);
+ await writeFile(foreign, savedPreview);
+ await expect(early.publish()).rejects.toThrow(
+ "original deployment and locale",
+ );
+ await expect(prepare()).rejects.toThrow("original deployment and locale");
+ const opaque = await readVendorFindings(f.file);
+ const getter = opaque.findings[0]!.evidence.source_data!;
+ Object.defineProperty(getter, "Z", { get: () => 1 });
+ Object.defineProperty(getter, "toJSON", { value: () => ({ ...data, Z: 9 }) });
+ await expect(prepare(opaque)).rejects.toThrow("cannot be matched");
+ expect(f.posts).toHaveLength(1);
+ expect(await readFile(foreign, "utf8")).toBe(savedPreview);
+ const distinct = await prepare(JSON.parse(JSON.stringify(opaque)));
+ expect(distinct.preview.resumed).toBe(false);
+
+ await rename(foreign, join(f.directory, localPreviewFilename()));
+ const retry = await prepare();
+ expect(retry.preview.resumed).toBe(true);
+ expect<unknown[]>(retry.preview.requests).toEqual(previewCheckpoint.requests);
+ const completed = await retry.publish();
+ expect(completed.verified).toBe(1);
+ expect<unknown[]>(completed.receipts).toEqual(previewCheckpoint.receipts);
+ expect(f.posts).toHaveLength(1); // Only the original import seeded by the fixture.
+ const fresh = await prepare();
+ expect(fresh.preview.resumed).toBe(false);
+ expect(fresh.preview.requests[0]!.items[0]!.evidence.source_data?.["Z"]).toBe(
+ 7,
+ );
+});
+
test.each(["distinct", "duplicate"] as const)(
"a migrated preview checkpoint completes readback with a %s older file",
async (kind) => {
@@ -2747,6 +2793,15 @@ test("concurrent legacy inventory resumers retain the acknowledged request after
newer.evidence.description =
"Newer Cloud evidence after the first resumer completed";
f.reports.set("sast:occurrence-1", newer);
+ const otherPath = join(
+ directory,
+ `${hash("sha256", "other-deployment")}.pending.json`,
+ );
+ const projectionless = JSON.stringify({
+ accountId: saved.accountId,
+ requests: saved.requests,
+ });
+ await writeFile(otherPath, projectionless, { mode: 0o600 });
releaseSecond.resolve();
const trailing = await second;
const result = await trailing.publish();
@@ -2756,6 +2811,7 @@ test("concurrent legacy inventory resumers retain the acknowledged request after
expect(result.receipts).toEqual(saved.receipts);
expect(result.verified).toBe(1);
expect(f.reports.get("sast:occurrence-1")).toEqual(newer);
+ expect(await readFile(otherPath, "utf8")).toBe(projectionless);
} finally {
secondAtLock.resolve();
releaseSecond.resolve();