OpenAI's Codex Security CLI and TypeScript SDK for finding, validating, and fixing security vulnerabilities. npm: https://www.npmjs.com/package/@openai/codex-security
TypeScript ai-securityapplication-securityclicode-scanningcodexcodex-securitycybersecuritydevsecopsnodejsnpmopenaisdksecuritytypescriptvulnerability-scanning
fix!: classify severity with a default rubric and configurable reviews (#1765)
Kyle Brown8c00f36cda71
diff --git a/plugins/codex-security/mcp-app/src/workbench/severity.ts b/plugins/codex-security/mcp-app/src/workbench/severity.tsindex d3f53e4f8..ca0c18fa8 100644--- a/plugins/codex-security/mcp-app/src/workbench/severity.ts+++ b/plugins/codex-security/mcp-app/src/workbench/severity.ts@@ -5,6 +5,7 @@ import type { Finding } from "./findings";import { transaction } from "./transaction";const fields = {+ classificationRuns: "classification_runs",findingId: "finding_id",occurrenceId: "occurrence_id",inputSha256: "input_sha256",@@ -20,7 +21,10 @@ const fields = {reviewTrigger: "review_trigger",} as const;-type Assessment = Record<keyof typeof fields, string | null>;+type Assessment = Record<+ Exclude<keyof typeof fields, "classificationRuns">,+ string | null+> & { classificationRuns?: number | null };export type SeverityCheckpoint =| {@@ -30,6 +34,7 @@ export type SeverityCheckpoint =assessedAt: string;rubricSha256: string | null;knowledgeBaseSha256: string | null;+ classificationRuns?: number;}| {action: "save";@@ -50,7 +55,12 @@ export function assessments(: "scan_severity_assessments";const scope = scanId === undefined ? "" : "WHERE assessment.scan_id = ?";// JSON preserves embedded NULs across Node 22 SQLite TEXT results.+ const hasRunCount = database+ .prepare(`PRAGMA table_info(${table})`)+ .all()+ .some((column) => column.name === "classification_runs");const projection = Object.entries(fields)+ .filter(([key]) => key !== "classificationRuns" || hasRunCount).map(([key, column]) => `'${key}', assessment.${column}`).join(", ");// Keep selected IDs first so each assessment is a primary-key lookup.@@ -84,13 +94,14 @@ export function severityCheckpoint(database.prepare(`INSERT INTO scan_severity_classifications (- scan_id, finding_ids_json, assessed_at, rubric_sha256, knowledge_base_sha256- ) VALUES (?, ?, ?, ?, ?)+ scan_id, finding_ids_json, assessed_at, rubric_sha256, knowledge_base_sha256, classification_runs+ ) VALUES (?, ?, ?, ?, ?, ?)ON CONFLICT(scan_id) DO UPDATE SETfinding_ids_json = excluded.finding_ids_json,assessed_at = excluded.assessed_at,rubric_sha256 = excluded.rubric_sha256,- knowledge_base_sha256 = excluded.knowledge_base_sha256`,+ knowledge_base_sha256 = excluded.knowledge_base_sha256,+ classification_runs = excluded.classification_runs`,).run(payload.scanId,@@ -98,6 +109,7 @@ export function severityCheckpoint(payload.assessedAt,payload.rubricSha256,payload.knowledgeBaseSha256,+ payload.classificationRuns ?? null,);// Cache hits are not saved again by the classifier. Keep each scan's own assessments.database@@ -121,10 +133,15 @@ export function severityCheckpoint(const values = Object.fromEntries(Object.entries(fields).map(([key, column]) => [column,- assessment[key as keyof Assessment],+ assessment[key as keyof Assessment] ?? null,]),);- requireSqliteText([finding.findingId, ...Object.values(values)]);+ requireSqliteText([+ finding.findingId,+ ...Object.values(values).filter(+ (value): value is string => typeof value === "string",+ ),+ ]);// External scan directories may not have been indexed on this machine.if (!database@@ -191,11 +208,16 @@ export function readSeverityClassification(.get())return {};+ const hasRunCount = database+ .prepare("PRAGMA table_info(scan_severity_classifications)")+ .all()+ .some((column) => column.name === "classification_runs");const row = database.prepare(`SELECT json_object('findingIds', json(finding_ids_json), 'assessedAt', assessed_at,'rubricSha256', rubric_sha256, 'knowledgeBaseSha256', knowledge_base_sha256+ ${hasRunCount ? ", 'classificationRuns', classification_runs" : ""}) AS value FROM scan_severity_classifications WHERE scan_id = ?`,).get(scanId);diff --git a/plugins/codex-security/mcp-app/tests/test_workbench_severity.ts b/plugins/codex-security/mcp-app/tests/test_workbench_severity.tsindex e1ee0cb36..af19fdf11 100644--- a/plugins/codex-security/mcp-app/tests/test_workbench_severity.ts+++ b/plugins/codex-security/mcp-app/tests/test_workbench_severity.ts@@ -43,6 +43,7 @@ function save(evidence: { count: 18446744073709551617n },},assessment: {+ classificationRuns: null,findingId,occurrenceId: `occurrence-${findingId}`,inputSha256: "evidence-digest",@@ -77,6 +78,7 @@ function begin(test("severity checkpoints retain external findings and each scan's cached assessments", (t) => {const database = open(t);const payload = save("finding\0suffix", "scan-a");+ payload.assessment.classificationRuns = 10;for (const field of ["occurrenceId","inputSha256",diff --git a/plugins/codex-security/shared/workbench-migrations.json b/plugins/codex-security/shared/workbench-migrations.jsonindex 0a18f1649..1e6a725b0 100644--- a/plugins/codex-security/shared/workbench-migrations.json+++ b/plugins/codex-security/shared/workbench-migrations.json@@ -425,5 +425,14 @@"CREATE TRIGGER record_deep_scan_worker_thread_insert\nAFTER INSERT ON deep_scan_workers\nWHEN NEW.sdk_thread_id IS NOT NULL\nBEGIN\n INSERT OR IGNORE INTO deep_scan_worker_threads (scan_id, worker_id, sdk_thread_id)\n VALUES (NEW.scan_id, NEW.id, NEW.sdk_thread_id);\nEND;","CREATE TRIGGER record_deep_scan_worker_thread_update\nAFTER UPDATE OF sdk_thread_id ON deep_scan_workers\nWHEN NEW.sdk_thread_id IS NOT NULL\nBEGIN\n INSERT OR IGNORE INTO deep_scan_worker_threads (scan_id, worker_id, sdk_thread_id)\n VALUES (NEW.scan_id, NEW.id, NEW.sdk_thread_id);\nEND;"]+ },+ {+ "version": 52,+ "name": "record severity classification review counts",+ "statements": [+ "ALTER TABLE finding_severity_assessments ADD COLUMN classification_runs INTEGER CHECK (classification_runs BETWEEN 1 AND 10);",+ "ALTER TABLE scan_severity_assessments ADD COLUMN classification_runs INTEGER CHECK (classification_runs BETWEEN 1 AND 10);",+ "ALTER TABLE scan_severity_classifications ADD COLUMN classification_runs INTEGER CHECK (classification_runs BETWEEN 1 AND 10);"+ ]}]diff --git a/plugins/codex-security/tests/test_workbench_setup_and_migrations.py b/plugins/codex-security/tests/test_workbench_setup_and_migrations.pyindex 59b149f4d..f1f7ec12d 100644--- a/plugins/codex-security/tests/test_workbench_setup_and_migrations.py+++ b/plugins/codex-security/tests/test_workbench_setup_and_migrations.py@@ -93,6 +93,7 @@ EXPECTED_MIGRATIONS = [(49, "index finding issue receipt ownership"),(50, "persist authorized source excerpt scopes"),(51, "preserve deep scan worker thread history"),+ (52, "record severity classification review counts"),]diff --git a/sdk/typescript/README.md b/sdk/typescript/README.mdindex acaa4ed04..84d15ab8b 100644--- a/sdk/typescript/README.md+++ b/sdk/typescript/README.md@@ -612,8 +612,11 @@ and [Azure Pipelines](https://github.com/openai/codex-security/blob/main/example### Classify finding severity-Use `classify-severity` with your own rubric to assess saved findings without-changing their original severity or evidence. The SDK exposes `classifySeverity()`,+Use `classify-severity` to assess saved findings with the built-in severity rubric+or an explicit custom rubric, without changing their original severity or evidence.+Each finding receives three independent reviews by default; the most common severity wins,+with the highest severity breaking ties among the top vote counts.+Use `--runs N` to request 1–10 reviews (default: 3). The SDK exposes `classifySeverity()`,`classifyScanSeverity()`, and `classifyScanDirectorySeverity()`. See[severity classification](docs/cli.md#classify-finding-severity) for selection,checkpointing, and publication behavior.diff --git a/sdk/typescript/docs/classify-severity-records.md b/sdk/typescript/docs/classify-severity-records.mdindex 07abf11d3..a1d2cb29c 100644--- a/sdk/typescript/docs/classify-severity-records.md+++ b/sdk/typescript/docs/classify-severity-records.md@@ -3,7 +3,7 @@`codex-security classify-severity --records` accepts canonical reports without arepository, saved scan, synthetic provenance, local Codex login, or filesystemstate. It uses the same bidirectional JSON-RPC transport as [record deduplication](dedupe-records.md).-Use `--records` alone; saved-scan classification remains unchanged.+Use `--records` alone or with `--runs N`; model execution remains host-provided.The SDK exports `classifySeverityRecords`, `classifySeverityRecordsInputSchema`,and their input, result, and review-request types. The protocol and input have@@ -25,6 +25,15 @@ Send one newline-delimited JSON-RPC request with `method: "run"` and these param}```+The `rubric` field is optional. Omit it to classify with the+[built-in severity rubric](../src/default-severity-rubric.ts), using the same configured+independent host reviews and usage accounting. The result echoes its built-in+revision (currently `1`) and exact Markdown SHA-256. Supplying a custom rubric+replaces the default completely and preserves the supplied revision/hash binding.+Do not reuse cached results across differing rubric hashes; preserve the default+snapshot from the review prompt alongside its returned binding. Default revisions+are local to the bundled policy, not workspace rubric revisions.+`findingJson` is the exact JSON text of an object with nonblank `findingId`,`title`, and `summary`. `findingId` must equal the canonical `id`. Optional`severity.level` uses `critical`, `high`, `medium`, `low`, or `informational`.@@ -36,7 +45,7 @@ normalizing newlines, whitespace, Unicode, object key order, or JSON numbers.Compute `inputSha256` from `findingJson`, not from its parsed/reserialized object.For example, Node uses `createHash("sha256").update(text, "utf8").digest("hex")`;Python uses `hashlib.sha256(text.encode("utf-8")).hexdigest()`.-The CLI recomputes both hashes before requesting any review. Rubric revisions are+The CLI recomputes supplied hashes before requesting any review. Rubric revisions arepositive safe integers; record revisions are nonnegative safe integers. Duplicatecanonical IDs and invalid records reject the entire input before host work.The implementation snapshots caller input before awaiting model execution and@@ -45,8 +54,10 @@ numeric evidence that JavaScript cannot represent exactly.## Host execution contract-For each record the CLI emits `method: "review.run"`, with a unique request ID-and params containing `requestId`, `stage: "severity-classification"`, explicit+The `classificationRuns` SDK option or CLI `--runs N` flag accepts+an integer from 1 through 10 (default: 3), validated before any host work.+For each record the CLI emits that many sequential `method: "review.run"` requests,+each with a unique request ID and params containing `requestId`, `stage: "severity-classification"`, explicit`model` and `effort`, `trustedInstructions`, untrusted report/rubric `prompt`, andan output JSON `schema`. The host must authorize the requested model and effort,create a fresh independent review context, preserve the instruction/data boundary,@@ -61,7 +72,14 @@ object. Do not wrap the assessment in a string or an array. An assessment contai`rubricLabel`, `rationale`, separate `confidence`, and nullable `reviewTrigger`.Excluded assessments require null level and label; assessed ones require both.-The host owns model budgets, transport limits, timeouts, and session cancellation.+The reviews receive identical inputs and do not see earlier answers. The+most common normalized severity wins; the highest among tied top counts wins+(`critical` > `high` > `medium` > `low` > `informational`). Exclusion is a separate+vote: exclusions can win by count, but lose ties to assessed severities. The first answer matching the winner supplies all assessment fields.++The host owns model budgets, usage accounting, transport limits, timeouts, and+session cancellation. Account for every review request, including work preceding+a later failure; classification results do not aggregate or report token usage.Use the transport's `cancel` notification with the active run ID and also stopany remote execution. EOF or a broken pipe means disconnection, not successfulcompletion. No review is retried automatically, including after an ambiguous@@ -70,20 +88,20 @@ host execution error.## Results and persistenceA completed result contains `version: 1`, `status: "completed"`, the rubric's-`revision` and `sha256`, and `assessments`. Each assessment replaces model-provided+`revision` and `sha256`, `classificationRuns`, and `assessments`. Each assessment replaces model-provided`findingId` with the verified canonical `id`, `revision`, and `inputSha256`.The original report and severity are never rewritten.Invalid or mismatched model output and host failures return `status: "unresolved"`,an empty `assessments` array, and an error with `recordId` and stable code-`invalid_assessment` or `review_failed`. Earlier assessments in that invocation-are discarded. Invalid input, protocol errors, and cancellation use JSON-RPC error-responses. Exit codes are 0 for completion, 1 for unresolved review, and 2 for+`invalid_assessment` or `review_failed`. All requested reviews must succeed; partial+votes and earlier assessments in that invocation are discarded. Invalid input,+protocol errors, and cancellation use JSON-RPC error responses. Exit codes are 0 for completion, 1 for unresolved review, and 2 forinput/protocol failure (130/143 for SIGINT/SIGTERM).Prefer one canonical record per workflow invocation when the host needs independentretries and partial success. The host should retain immutable rubric/report-snapshots, compare every echoed binding, and conditionally persist only computed+snapshots, compare every echoed binding including `classificationRuns`, and conditionally persist only computedprioritization fields if the expected canonical version and rubric still apply.Keep manual edits and original severity intact. Repeating the same inputs bindsresults to the same hashes; it does not promise deterministic model answers.diff --git a/sdk/typescript/docs/cli.md b/sdk/typescript/docs/cli.mdindex 75b0ca7e4..799f22a0f 100644--- a/sdk/typescript/docs/cli.md+++ b/sdk/typescript/docs/cli.md@@ -1283,16 +1283,31 @@ codex-security classify-severity --scan latest --rubric /path/to/policy.md --jsocodex-security classify-severity --scan-dir /path/to/completed-scan --json```-For repository-free canonical records, use `classify-severity --records` alone.+For repository-free canonical records, use `classify-severity --records` (optionally with `--runs N`).The [records protocol](classify-severity-records.md) accepts immutable Markdown andreport snapshots and delegates model execution to the host.For saved findings, use exactly one of `--scan` (ID, prefix, or `latest`) and `--scan-dir`.-Omitting `--rubric` inherits original severity without model calls. Rubrics and-repeatable `--knowledge-base` context accept text, PDF, DOCX, and directories.-Each finding is classified in a separate read-only turn using its report and-supplied context, without tools or source inspection. `--model` and `--effort`-override Codex's configured model and the default medium effort.+Omitting `--rubric` runs classification with the [built-in severity rubric](../src/default-severity-rubric.ts).+It evaluates attacker access, boundary crossing, additional harm, and evidence for+Critical, High, Medium, Low, and Informational findings. Confidence remains separate+from severity; missing proof alone does not justify a downgrade. An explicit+`--rubric` replaces the default policy completely. Rubrics and repeatable+`--knowledge-base` context accept text, PDF, DOCX, and directories.+Each finding is classified three times by default. `--runs N` accepts+an integer from 1 through 10; SDK options use `classificationRuns` with the same+default and bounds. Each review runs in an independent read-only turn using the+same report, rubric, and supplied context, without tools or source inspection.+The most common normalized severity wins; the highest among tied top counts wins+(`critical` > `high` > `medium` > `low` > `informational`). Exclusion is a separate+vote: exclusions can win by count, but lose ties to assessed severities. The first answer matching the winner supplies its label, rationale,+confidence, and review trigger. All requested reviews must succeed; failures or invalid+answers fail that finding without saving a partial vote. This uses the requested number of model+calls per uncached finding (three by default), including when no custom rubric is supplied, and+requires the same Codex authentication and model access as custom classification.+Codex retains usage for each independent turn. There is no fallback to scanner+severity if classification fails.+`--model` and `--effort` override Codex's configured model and the default medium effort.Assessments contain `decision` (`assessed` or `excluded`), normalized `level`,the original `rubricLabel`, rationale, confidence, and a `reviewTrigger` for@@ -1300,10 +1315,17 @@ missing facts that could change the result. Exclusions have a null level.Finding/occurrence IDs and input hashes bind each assessment to its evidence.Successful assessments checkpoint immediately in local SQLite. Retries reuse-matching evidence, rubric, and context hashes; failed/canceled runs keep completed-work. Use `--reprocess` to replace assessments, including when changing only model-or effort. A successful run also writes `severity-classification.json`, but-publication reads the database. Original findings and sealed artifacts remain+matching evidence, rubric, context hashes, and classification run count; failed/canceled runs keep completed+work. Default-rubric assessments use `source: "rubric"` and a content hash; former+no-rubric inherited assessments have no rubric hash and are automatically+reclassified on the next classification run. Changes to the bundled policy or+switching between default and custom policies also invalidate matching checkpoints.+The result and checkpoint record `classificationRuns`; changing the count, or+reusing an older checkpoint with no recorded count, triggers fresh classification.+Older results remain readable; publication does not silently run a model.+Use `--reprocess` to replace assessments, including when changing only model+or effort. A successful+run also writes `severity-classification.json`, but publication reads the database. Original findings and sealed artifacts remainunchanged.Repeat `--finding-id ID` for a subset from the selected scan. Dedupe's@@ -1374,8 +1396,9 @@ await publishScan(scanDirectory, {````classifySeverity()` accepts `findingId`, `title`, `summary`, and available-evidence/metadata. Imported reports may lack severity or occurrence IDs; missing-severity requires a rubric. Scan wrappers persist checkpoints (including for+evidence/metadata. Imported reports may lack severity or occurrence IDs; omitting+`rubricPath` uses the built-in policy even when a scanner severity is present.+Scan wrappers persist checkpoints (including forexternal directories), accept `reprocess: true`, and treat `findingIds: []` as anempty selection. Use the same `CODEX_SECURITY_STATE_DIR` for classification andpublication. Older JSON-only assessments need reclassification once. Alldiff --git a/sdk/typescript/scripts/package-dist-files.mjs b/sdk/typescript/scripts/package-dist-files.mjsindex 897be9037..e57af79bd 100644--- a/sdk/typescript/scripts/package-dist-files.mjs+++ b/sdk/typescript/scripts/package-dist-files.mjs@@ -20,6 +20,7 @@ const distModules = ["classify-scan-severity","severity-store","severity-review",+ "default-severity-rubric","cloud-publish","codex-prompt","codex-sdk-environment",diff --git a/sdk/typescript/src/classify-severity-records.ts b/sdk/typescript/src/classify-severity-records.tsindex 335d29cd5..3361f8f94 100644--- a/sdk/typescript/src/classify-severity-records.ts+++ b/sdk/typescript/src/classify-severity-records.ts@@ -1,9 +1,12 @@import { SCAN_SEVERITIES } from "./scan-settings.js";+import { defaultSeverityRubric } from "./default-severity-rubric.js";import { createHash, randomUUID } from "node:crypto";import { z } from "zod";import { abortable } from "./targets.js";import {parseSeverityDecision,+ reviewSeverity,+ classificationRunsSchema,severityDecisionSchema,severityReviewInstructions,type SeverityDecision,@@ -41,7 +44,8 @@ export const classifySeverityRecordsInputSchema = zmarkdown: text,sha256: digest,})- .strict(),+ .strict()+ .optional(),model: text,effort: z.enum(["minimal",@@ -80,6 +84,8 @@ export interface SeverityClassificationReviewRequest {}export interface ClassifySeverityRecordsOptions {+ /** Independent reviews per record: integer 1–10, default 3. */+ classificationRuns?: number;reviewRunner: {run(request: SeverityClassificationReviewRequest,@@ -99,6 +105,7 @@ export type CanonicalSeverityAssessment = Omit<};interface RecordsResultBinding {+ classificationRuns: number;version: 1;rubric: { revision: number; sha256: string };}@@ -130,8 +137,16 @@ export async function classifySeverityRecords(options: ClassifySeverityRecordsOptions,): Promise<ClassifySeverityRecordsResult> {options.signal?.throwIfAborted();+ const classificationRuns = classificationRunsSchema+ .default(3)+ .parse(options.classificationRuns);const data = classifySeverityRecordsInputSchema.parse(structuredClone(input));- if (sha256(data.rubric.markdown) !== data.rubric.sha256)+ const rubric = data.rubric ?? {+ revision: 1,+ markdown: defaultSeverityRubric,+ sha256: sha256(defaultSeverityRubric),+ };+ if (sha256(rubric.markdown) !== rubric.sha256)throw new Error("Rubric SHA-256 does not match its exact UTF-8 Markdown.");const ids = new Set<string>();for (const record of data.records) {@@ -147,49 +162,40 @@ export async function classifySeverityRecords();}const binding: RecordsResultBinding = {+ classificationRuns,version: 1,- rubric: { revision: data.rubric.revision, sha256: data.rubric.sha256 },+ rubric: { revision: rubric.revision, sha256: rubric.sha256 },};const assessments: CanonicalSeverityAssessment[] = [];for (const record of data.records) {options.signal?.throwIfAborted();- let response: unknown;+ let code: "review_failed" | "invalid_assessment" = "review_failed";try {- response = await abortable(() => {- options.signal?.throwIfAborted();- return options.reviewRunner.run(- {- requestId: randomUUID(),- stage: "severity-classification",- model: data.model,- effort: data.effort,- trustedInstructions: severityReviewInstructions,- prompt: `{"rubric":${JSON.stringify([data.rubric.markdown])},"knowledgeBase":null,"finding":${record.findingJson}}`,- schema: z.toJSONSchema(severityDecisionSchema),- },- { signal: options.signal },- );- }, options.signal);- } catch (error) {- options.signal?.throwIfAborted();- return {- ...binding,- status: "unresolved",- assessments: [],- error: {- recordId: record.id,- code: "review_failed",- message:- error instanceof Error ? error.message : "Host review failed.",+ const { findingId: _findingId, ...decision } = await reviewSeverity(+ async () => {+ code = "review_failed";+ const response = await abortable(() => {+ options.signal?.throwIfAborted();+ return options.reviewRunner.run(+ {+ requestId: randomUUID(),+ stage: "severity-classification",+ model: data.model,+ effort: data.effort,+ trustedInstructions: severityReviewInstructions,+ prompt: `{"rubric":${JSON.stringify([rubric.markdown])},"knowledgeBase":null,"finding":${record.findingJson}}`,+ schema: z.toJSONSchema(severityDecisionSchema),+ },+ { signal: options.signal },+ );+ }, options.signal);+ options.signal?.throwIfAborted();+ code = "invalid_assessment";+ return parseSeverityDecision(response, record.id);},- };- }- options.signal?.throwIfAborted();- try {- const { findingId: _findingId, ...decision } = parseSeverityDecision(- response,- record.id,+ classificationRuns,);+ options.signal?.throwIfAborted();assessments.push({...decision,id: record.id,@@ -197,15 +203,20 @@ export async function classifySeverityRecords(inputSha256: record.inputSha256,});} catch (error) {+ options.signal?.throwIfAborted();return {...binding,status: "unresolved",assessments: [],error: {recordId: record.id,- code: "invalid_assessment",+ code,message:- error instanceof Error ? error.message : "Invalid assessment.",+ error instanceof Error+ ? error.message+ : code === "review_failed"+ ? "Host review failed."+ : "Invalid assessment.",},};}diff --git a/sdk/typescript/src/classify-severity.ts b/sdk/typescript/src/classify-severity.tsindex 8fbf01e0f..2e65ffdab 100644--- a/sdk/typescript/src/classify-severity.ts+++ b/sdk/typescript/src/classify-severity.ts@@ -1,4 +1,4 @@-import { SCAN_SEVERITIES } from "./scan-settings.js";+import { defaultSeverityRubric } from "./default-severity-rubric.js";import { findingEntry } from "./value.js";import { z } from "zod";import { CodexSecurityError } from "./errors.js";@@ -13,6 +13,8 @@ import {severityDecisionSchema as decisionSchema,severityReviewInstructions,parseSeverityDecision,+ reviewSeverity,+ classificationRunsSchema,} from "./severity-review.js";import { CODEX_SECURITY_THREAD_SOURCES } from "./thread-source.js";@@ -33,8 +35,10 @@ export interface ClassifySeverityOptions extends Pick<| "signal"| "workingDirectory"> {- /** Classification policy. Omit to inherit existing severity without a model call. */+ /** Classification policy override. Omit to use the built-in severity rubric. */rubricPath?: string;+ /** Independent reviews per finding: integer 1–10, default 3. */+ classificationRuns?: number;/** Supporting evidence, separate from classification policy. */knowledgeBasePaths?: readonly string[];/** @internal Test client for the shared read-only runtime. */@@ -50,6 +54,8 @@ export interface SeverityAssessment extends z.infer<typeof decisionSchema> {export interface SeverityClassification {schemaVersion: 1;assessedAt: string;+ /** Absent on legacy results whose review count was not recorded. */+ classificationRuns?: number;rubricSha256: string | null;knowledgeBaseSha256: string | null;assessments: SeverityAssessment[];@@ -65,7 +71,6 @@ export interface SeverityClassificationCheckpoint {): Promise<void>;}-const levelSchema = z.enum(SCAN_SEVERITIES);const textSchema = z.string().min(1)@@ -75,6 +80,7 @@ const digestSchema = z.string().regex(/^[a-f0-9]{64}$/u);export const severityClassificationSchema = z.strictObject({schemaVersion: z.literal(1),assessedAt: z.string().datetime(),+ classificationRuns: classificationRunsSchema.optional(),rubricSha256: digestSchema.nullable(),knowledgeBaseSha256: digestSchema.nullable(),assessments: z.array(@@ -102,6 +108,9 @@ export async function classifySeverityInternal(checkpoint?: SeverityClassificationCheckpoint,): Promise<SeverityClassification> {options.signal?.throwIfAborted();+ const classificationRuns = classificationRunsSchema+ .default(3)+ .parse(options.classificationRuns);const ids = new Set<string>();for (const finding of findings) {if (!finding.findingId?.trim() || ids.has(finding.findingId)) {@@ -113,15 +122,16 @@ export async function classifySeverityInternal(}const rubric =options.rubricPath === undefined- ? null+ ? [defaultSeverityRubric]: await readDocuments([options.rubricPath], options.signal);const knowledge = options.knowledgeBasePaths?.length? await readDocuments(options.knowledgeBasePaths, options.signal): null;const result: SeverityClassification = {schemaVersion: 1,+ classificationRuns,assessedAt: new Date().toISOString(),- rubricSha256: rubric === null ? null : workflowDigest(rubric),+ rubricSha256: workflowDigest(rubric),knowledgeBaseSha256: knowledge === null ? null : workflowDigest(knowledge),assessments: [],};@@ -137,31 +147,14 @@ export async function classifySeverityInternal(result.assessments.push(previous);continue;}- let decision: z.infer<typeof decisionSchema>;- if (rubric === null) {- const parsed = levelSchema.safeParse(finding.severity?.level);- if (!parsed.success) {- throw new CodexSecurityError(- `Finding ${finding.findingId} has no existing severity; supply a rubric.`,- );- }- decision = {- findingId: finding.findingId,- decision: "assessed",- level: parsed.data,- rubricLabel: null,- rationale:- finding.severity?.rationale?.trim() ||- "Inherited the finding's existing severity.",- confidence: null,- reviewTrigger: finding.severity?.changeConditions?.trim() || null,- };- } else {+ const prompt = [+ severityReviewInstructions,+ JSON.stringify({ rubric, knowledgeBase: knowledge, finding }),+ ].join("\n\n");+ const decision = await reviewSeverity(async () => {+ options.signal?.throwIfAborted();const response = await runReadOnlyCodex(- [- severityReviewInstructions,- JSON.stringify({ rubric, knowledgeBase: knowledge, finding }),- ].join("\n\n"),+ prompt,z.toJSONSchema(decisionSchema),options,{@@ -172,22 +165,20 @@ export async function classifySeverityInternal();options.signal?.throwIfAborted();try {- decision = parseSeverityDecision(- JSON.parse(response),- finding.findingId,- );+ return parseSeverityDecision(JSON.parse(response), finding.findingId);} catch (error) {throw new CodexSecurityError("Severity classification returned an invalid assessment.",{ cause: error },);}- }+ }, classificationRuns);+ options.signal?.throwIfAborted();const assessment: SeverityAssessment = {...decision,occurrenceId: finding.occurrenceId ?? null,inputSha256,- source: rubric === null ? "existing-severity" : "rubric",+ source: "rubric",};await checkpoint?.save(finding, assessment, result);result.assessments.push(assessment);diff --git a/sdk/typescript/src/cli.ts b/sdk/typescript/src/cli.tsindex 965544acb..27e52285f 100644--- a/sdk/typescript/src/cli.ts+++ b/sdk/typescript/src/cli.ts@@ -1,4 +1,5 @@#!/usr/bin/env node+import { classificationRunsSchema } from "./severity-review.js";import { workbenchArgs } from "./workbench-arguments.js";import { listenForAbort } from "./cli-signals.js";import { appServerCyberAccessProgram } from "./cli-daybreak.js";@@ -50,6 +51,7 @@ import { Readable, Writable as NodeWritable } from "node:stream";import { fileURLToPath, pathToFileURL } from "node:url";import {isDeepStrictEqual,+ parseArgs,promisify,stripVTControlCharacters,} from "node:util";@@ -1877,7 +1879,29 @@ export async function main(!argv.includes("-h") &&!argv.includes("--schema")) {- if (argv.length !== 2 || (argv[0] === "scan" && argv[1] !== "--host")) {+ let classificationRuns: number | undefined;+ if (argv[0] === "classify-severity") {+ try {+ const { values } = parseArgs({+ args: argv.slice(1),+ options: {+ records: { type: "boolean" },+ runs: { type: "string" },+ },+ });+ classificationRuns = classificationRunsSchema.parse(+ values.runs === undefined ? 3 : Number(values.runs),+ );+ } catch {+ errorOutput.write(+ "codex-security: Use --records with only --runs (an integer from 1 to 10; default: 3).\n",+ );+ return 2;+ }+ } else if (+ argv.length !== 2 ||+ (argv[0] === "scan" && argv[1] !== "--host")+ ) {errorOutput.write(argv[0] === "scan"? "codex-security: --host must be used alone with scan.\n"@@ -1901,10 +1925,10 @@ export async function main(dependencies.recordsInput ?? process.stdin,output,(value, options) =>- classifySeverityRecords(- value as ClassifySeverityRecordsInput,- options,- ),+ classifySeverityRecords(value as ClassifySeverityRecordsInput, {+ ...options,+ classificationRuns,+ }),controller.signal,): await runProtocol(@@ -4116,15 +4140,20 @@ export async function main(}).command("classify-severity", {description: "Assess severity for saved findings or canonical records.",- hint: "Use --rubric for saved findings or --records alone for host-provided classification.",+ hint: "Use --rubric to override the built-in policy or --records for host-provided classification.",destructive: true,mcp: false,options: z.object({+ runs: classificationRunsSchema+ .default(3)+ .describe(+ "Number of independent classification runs (1–10; default: 3).",+ ),records: z.boolean().default(false).describe(- "Classify canonical records through host-provided reviews over version 1 JSON-RPC; use alone.",+ "Classify canonical records through host-provided reviews over version 1 JSON-RPC; accepts --runs.",),reprocess: z.boolean()@@ -4141,7 +4170,7 @@ export async function main(rubric: optionValue("--rubric").optional().describe(- "Classification policy document; omit to inherit existing severity without a model call.",+ "Classification policy document; omit to classify with the built-in severity rubric.",),knowledgeBase: z.array(optionValue("--knowledge-base"))@@ -4192,6 +4221,7 @@ export async function main(findingIds:options.findingId.length === 0 ? undefined : options.findingId,reprocess: options.reprocess,+ classificationRuns: options.runs,model: options.model,reasoningEffort: options.effort,};diff --git a/sdk/typescript/src/default-severity-rubric.ts b/sdk/typescript/src/default-severity-rubric.tsnew file mode 100644index 000000000..6c1c5bb20--- /dev/null+++ b/sdk/typescript/src/default-severity-rubric.ts@@ -0,0 +1,66 @@+/** Built into the CLI and SDK so classification needs no external policy file. */+export const defaultSeverityRubric = `# Severity guidance++Judge the attacker's actual starting access, the security boundary crossed, and+the additional unauthorized harm. Use the supplied report and evidence; do not+perform new validation. Keep confidence separate from severity and identify the+missing fact that could change the assessment. Missing proof alone does not+establish that a broader attack is impossible; do not invent an attack path or+downgrade merely because exploitation was not demonstrated.++## Critical++Reserve Critical for consequential, strongly supported compromise meeting all+three conditions:+- Access: an unauthenticated internet attacker, ordinary external user, or+ publicly admitted untrusted workload can exploit the interface without+ employee-only access, an invitation, an approved business relationship, or+ compromise of a trusted supplier.+- Impact: crossing an important security boundary enables consequential access+ to protected data, privileged actions, meaningful code execution, workload+ escape, or comparable compromise of the affected system.+- Evidence: demonstrated exploitation or strong source-backed validation+ establishes the complete attack path.++Second-order attacks can qualify: an external attacker plants data that a trusted+user or system later processes, causing consequential unauthorized access or+privileged actions.++## High++Assign High when a realistic, evidenced path gives a lower-trust attacker a+substantial unauthorized capability they did not already possess. Internal+users, partners, invited collaborators, and admitted untrusted workloads can+qualify; unauthenticated internet access is not required. Consequences include+sensitive customer or business data or reusable credentials, account takeover,+meaningful cross-tenant access or actions as another user, control of production+workloads or deployments, privileged identities, encryption keys or important+isolation boundaries, or significant repeatable disruption of a critical or+shared service. A dangerous function, production label, internal access, or the+existence of a privileged credential alone is insufficient: identify the+lower-trust actor, boundary crossing, and consequential new capability.++## Medium++Assign Medium to a credible vulnerability with concretely bounded impact, such+as nonsecret operational information, narrow unauthorized changes, contained+disruption of a noncritical service, or isolated development or staging systems+without a consequential pivot. A fully demonstrated vulnerability can still be+Medium. Base a downgrade on verified constraints, not on an assumption that+missing evidence rules out broader harm.++## Low++Assign Low to a supported security weakness with minor additional unauthorized+harm and verified limits on its scope, sensitivity, or duration, such as a small+information exposure with little security value or a narrowly contained nuisance.+Do not use Low as a substitute for uncertainty about a potentially serious path.++## Informational++Assign Informational to a hardening opportunity or observation for which the+supplied evidence establishes no additional unauthorized capability or security+impact. Explain why it is informational rather than treating missing proof of+exploitation as proof of safety. Administrative records that do not describe a+security finding may be excluded; exclusion is distinct from severity.+`;diff --git a/sdk/typescript/src/severity-review.ts b/sdk/typescript/src/severity-review.tsindex 71d89542e..3d8aa1d9c 100644--- a/sdk/typescript/src/severity-review.ts+++ b/sdk/typescript/src/severity-review.ts@@ -2,6 +2,8 @@ import { SCAN_SEVERITIES } from "./scan-settings.js";import { z } from "zod";import { CodexSecurityError } from "./errors.js";+export const classificationRunsSchema = z.number().int().min(1).max(10);+const levelSchema = z.enum(SCAN_SEVERITIES);const textSchema = z.string()@@ -58,3 +60,25 @@ export function parseSeverityDecision();}}++/** Run independent reviews and retain the first complete answer for the winning level. */+export async function reviewSeverity(+ review: () => Promise<SeverityDecision>,+ classificationRuns = 3,+): Promise<SeverityDecision> {+ const runs = classificationRunsSchema.parse(classificationRuns);+ const decisions: SeverityDecision[] = [];+ for (let run = 0; run < runs; run++) decisions.push(await review());+ // Exclusion is a distinct vote; any assessed severity wins a tied vote.+ const levels = [...SCAN_SEVERITIES, null];+ const count = (level: SeverityDecision["level"]) =>+ decisions.filter((decision) => decision.level === level).length;+ return decisions.reduce((winner, decision) => {+ const difference = count(decision.level) - count(winner.level);+ return difference > 0 ||+ (difference === 0 &&+ levels.indexOf(decision.level) < levels.indexOf(winner.level))+ ? decision+ : winner;+ });+}diff --git a/sdk/typescript/src/severity-store.ts b/sdk/typescript/src/severity-store.tsindex 040d132fd..ab3a2cbfd 100644--- a/sdk/typescript/src/severity-store.ts+++ b/sdk/typescript/src/severity-store.ts@@ -41,6 +41,7 @@ export class SeverityStore {scanId,findingIds,assessedAt: result.assessedAt,+ classificationRuns: result.classificationRuns,rubricSha256: result.rubricSha256,knowledgeBaseSha256: result.knowledgeBaseSha256,});@@ -58,6 +59,7 @@ export class SeverityStore {finding,assessment: {...assessment,+ classificationRuns: result.classificationRuns,rubricSha256: result.rubricSha256,knowledgeBaseSha256: result.knowledgeBaseSha256,},@@ -85,6 +87,8 @@ export class SeverityStore {{schemaVersion: 1,assessedAt: saved["assessedAt"] as string,+ classificationRuns: (saved["classificationRuns"] ?? undefined) as+ number | undefined,rubricSha256: saved["rubricSha256"] as string | null,knowledgeBaseSha256: saved["knowledgeBaseSha256"] as string | null,},@@ -139,12 +143,15 @@ function matchingClassification(assessments: stored.filter((row) =>+ (row["classificationRuns"] ?? undefined) ===+ result.classificationRuns &&row["rubricSha256"] === result.rubricSha256 &&row["knowledgeBaseSha256"] === result.knowledgeBaseSha256,).map(({assessedAt: _assessedAt,+ classificationRuns: _runs,rubricSha256: _rubric,knowledgeBaseSha256: _knowledge,...assessmentdiff --git a/sdk/typescript/tests-ts/classify-scan-severity.test.ts b/sdk/typescript/tests-ts/classify-scan-severity.test.tsindex 5d1587d28..40a85efa2 100644--- a/sdk/typescript/tests-ts/classify-scan-severity.test.ts+++ b/sdk/typescript/tests-ts/classify-scan-severity.test.ts@@ -180,6 +180,7 @@ test("classification and saved assessments run with Node and no Python", async (const { scanId: _scanId, ...classification } =await classifyScanDirectorySeverity(scanDirectory, {environment: withoutPython,+ codex: recordingClassifier().codex,});expect(await readScanSeverityClassification(@@ -196,7 +197,10 @@ test("saved assessments retain embedded NULs in imported scan IDs", async () =>const { environment, scanDirectory, scanId, findings } =await fixture("imported\0scan");const { scanId: classifiedScanId, ...classification } =- await classifyScanDirectorySeverity(scanDirectory, { environment });+ await classifyScanDirectorySeverity(scanDirectory, {+ environment,+ codex: recordingClassifier().codex,+ });expect(classifiedScanId).toBe(scanId);expect(await readScanSeverityClassification(@@ -235,7 +239,7 @@ test("severity helpers skip Node executables in the caller's enclosing checkout"["--eval",`import { classifyScanDirectorySeverity } from ${JSON.stringify(entryPoint.href)};-await classifyScanDirectorySeverity(${JSON.stringify(scanDirectory)}, { environment: process.env });`,+await classifyScanDirectorySeverity(${JSON.stringify(scanDirectory)}, { environment: process.env, findingIds: [] });`,],{cwd,@@ -374,13 +378,17 @@ test("saved severity protects repository and caller tools across concurrent lookfunction recordingClassifier() {const calls: string[] = [];- const control = { failOn: "", excluded: false };+ const control = { failOn: "", failOnRun: 1, excluded: false };const codex = codexWithRun(async (prompt: string) => {const { finding } = JSON.parse(prompt.split("\n\n").at(-1)!) as {finding: Finding;};calls.push(finding.findingId);- if (finding.findingId === control.failOn)+ if (+ finding.findingId === control.failOn &&+ calls.filter((id) => id === finding.findingId).length ===+ control.failOnRun+ )throw new Error("Interrupted model call");return classifier(finding, control.excluded).startThread().run();});@@ -388,10 +396,11 @@ function recordingClassifier() {}test("checkpoints each finding, resumes missing work, and reprocesses only the selection", async () => {- const { environment, scanDirectory, rubricPath, findings } = await fixture();+ const { environment, scanDirectory, findings } = await fixture();const { codex, calls, control } = recordingClassifier();- const options = { environment, rubricPath, codex };+ const options = { environment, codex };control.failOn = findings[1]!.findingId;+ control.failOnRun = 3;await expect(classifyScanDirectorySeverity(scanDirectory, options),).rejects.toThrow("Interrupted model call");@@ -409,7 +418,7 @@ test("checkpoints each finding, resumes missing work, and reprocesses only the scontrol.failOn = "";calls.length = 0;const complete = await classifyScanDirectorySeverity(scanDirectory, options);- expect(calls).toEqual([findings[1]!.findingId]);+ expect(calls).toEqual(Array(3).fill(findings[1]!.findingId));expect(complete.assessments).toHaveLength(2);const { scanId: _scanId, ...assessment } = complete;expect(@@ -444,7 +453,7 @@ test("checkpoints each finding, resumes missing work, and reprocesses only the sreprocess: true,};const revised = await classifyScanDirectorySeverity(scanDirectory, selected);- expect(calls).toEqual([findings[0]!.findingId]);+ expect(calls).toEqual(Array(3).fill(findings[0]!.findingId));expect(revised.assessments[0]!.decision).toBe("excluded");const revisedRows = await query(environment,@@ -470,6 +479,7 @@ test("checkpoints each finding, resumes missing work, and reprocesses only the s).toHaveLength(1);control.failOn = findings[0]!.findingId;+ control.failOnRun = 2;await expect(classifyScanDirectorySeverity(scanDirectory, selected),).rejects.toThrow("Interrupted model call");@@ -555,10 +565,22 @@ test("migration leaves unindexed legacy assessments incomplete until reclassifieconst environment = first.environment;const { scanId, ...classification } = await classifyScanDirectorySeverity(first.scanDirectory,- { environment },+ { environment, codex: recordingClassifier().codex },);await query(environment, "DROP TABLE scan_severity_assessments");- await query(environment, "DELETE FROM schema_migrations WHERE version = 43");+ for (const table of [+ "finding_severity_assessments",+ "scan_severity_classifications",+ ]) {+ await query(+ environment,+ `ALTER TABLE ${table} DROP COLUMN classification_runs`,+ );+ }+ await query(+ environment,+ "DELETE FROM schema_migrations WHERE version IN (43, 52)",+ );expect(await readScanSeverityClassification(first.scanDirectory,@@ -567,14 +589,17 @@ test("migration leaves unindexed legacy assessments incomplete until reclassifieundefined,environment,),- ).toEqual(classification);+ ).toEqual({ ...classification, classificationRuns: undefined });expect(await query(environment,"SELECT version FROM schema_migrations WHERE version = 43",),).toEqual([]);- await classifyScanDirectorySeverity(second.scanDirectory, { environment });+ await classifyScanDirectorySeverity(second.scanDirectory, {+ environment,+ codex: recordingClassifier().codex,+ });await expect(readScanSeverityClassification(first.scanDirectory,@@ -585,8 +610,12 @@ test("migration leaves unindexed legacy assessments incomplete until reclassifie),).rejects.toThrow("incomplete");expect(- (await classifyScanDirectorySeverity(first.scanDirectory, { environment }))- .assessments,+ (+ await classifyScanDirectorySeverity(first.scanDirectory, {+ environment,+ codex: recordingClassifier().codex,+ })+ ).assessments,).toEqual(classification.assessments);expect(await query(@@ -609,17 +638,17 @@ test("changed rubric, context, or evidence invalidates matching checkpoints", ascalls.length = 0;await writeFile(rubricPath, "Assign Medium to bounded metadata reads.");await classifyScanDirectorySeverity(scanDirectory, options);- expect(calls).toHaveLength(2);+ expect(calls).toHaveLength(6);const context = join(root, "context.md");await writeFile(context, "The system contains operational counters.");calls.length = 0;const withContext = { ...options, knowledgeBasePaths: [context] };await classifyScanDirectorySeverity(scanDirectory, withContext);- expect(calls).toHaveLength(2);+ expect(calls).toHaveLength(6);calls.length = 0;await writeFile(context, "The counters include protected metadata.");await classifyScanDirectorySeverity(scanDirectory, withContext);- expect(calls).toHaveLength(2);+ expect(calls).toHaveLength(6);const findingPath = join(scanDirectory, "findings.json");const document = JSON.parse(@@ -641,7 +670,7 @@ test("changed rubric, context, or evidence invalidates matching checkpoints", as).rejects.toThrow("does not match");calls.length = 0;await classifyScanDirectorySeverity(scanDirectory, withContext);- expect(calls).toEqual([findings[0]!.findingId]);+ expect(calls).toEqual(Array(3).fill(findings[0]!.findingId));expect(await query(environment, "SELECT * FROM findings ORDER BY id"),).toEqual(originalFindings);@@ -772,7 +801,10 @@ test("exclusions and empty dedupe selections do not create tickets", async () =>test("failed or canceled reassessment leaves the last successful assessment intact", async () => {const { environment, scanDirectory, rubricPath, findings } = await fixture();- await classifyScanDirectorySeverity(scanDirectory, { environment });+ await classifyScanDirectorySeverity(scanDirectory, {+ environment,+ codex: recordingClassifier().codex,+ });const path = join(scanDirectory, "severity-classification.json");const before = await readFile(path);await expect(@@ -805,6 +837,7 @@ test("publication reads SQLite even when the JSON export is modified or symlinkeconst { environment, root, scanDirectory, findings } = await fixture();const result = await classifyScanDirectorySeverity(scanDirectory, {environment,+ codex: recordingClassifier().codex,});const path = join(scanDirectory, "severity-classification.json");await writeFile(path, JSON.stringify({ ...result, scanId: "other-scan" }));@@ -816,9 +849,10 @@ test("publication reads SQLite even when the JSON export is modified or symlinke})).issues,).toHaveLength(findings.length);- const stale = await classifySeverity([- { ...findings[0]!, summary: "Different report" },- ]);+ const stale = await classifySeverity(+ [{ ...findings[0]!, summary: "Different report" }],+ { codex: classifier(findings[0]!) },+ );await expect(prepareScanPublication(scanDirectory, {environment,@@ -838,7 +872,10 @@ test("publication reads SQLite even when the JSON export is modified or symlinke})).issues,).toHaveLength(findings.length);- await classifyScanDirectorySeverity(scanDirectory, { environment });+ await classifyScanDirectorySeverity(scanDirectory, {+ environment,+ codex: recordingClassifier().codex,+ });expect(await readFile(external, "utf8")).toBe(JSON.stringify(result));});@@ -916,7 +953,7 @@ test.each(["latest", "scan_prefix"])(const seen: string[][] = [];const result = await classifyScanSeverityInternal(selector,- { environment },+ { environment, codex: recordingClassifier().codex },{currentDirectory: () => scanDirectory,runWorkbench: async (args): Promise<JsonObject> => {@@ -950,7 +987,10 @@ test.each(["latest", "scan_prefix"])(test("migrates existing databases without changing findings and reads older state without writes", async () => {const { environment, scanDirectory } = await fixture();- await classifyScanDirectorySeverity(scanDirectory, { environment });+ await classifyScanDirectorySeverity(scanDirectory, {+ environment,+ codex: recordingClassifier().codex,+ });const original = await query(environment,"SELECT * FROM findings ORDER BY id",@@ -960,7 +1000,7 @@ test("migrates existing databases without changing findings and reads older statawait query(environment, "DROP TABLE scan_severity_classifications");await query(environment,- "DELETE FROM schema_migrations WHERE version IN (41, 43)",+ "DELETE FROM schema_migrations WHERE version IN (41, 43, 52)",);expect((@@ -976,7 +1016,10 @@ test("migrates existing databases without changing findings and reads older stat"SELECT version FROM schema_migrations WHERE version = 41",),).toEqual([]);- await classifyScanDirectorySeverity(scanDirectory, { environment });+ await classifyScanDirectorySeverity(scanDirectory, {+ environment,+ codex: recordingClassifier().codex,+ });expect(await query(environment,@@ -1020,7 +1063,7 @@ test("classifying another scan preserves both recurring-finding assessments", as};await classifyScanDirectorySeverity(first.scanDirectory, options);await classifyScanDirectorySeverity(second.scanDirectory, options);- expect(calls).toHaveLength(4);+ expect(calls).toHaveLength(12);calls.length = 0;for (const scan of [first, second, first, second]) {expect(@@ -1035,3 +1078,143 @@ test("classifying another scan preserves both recurring-finding assessments", as}expect(calls).toEqual([]);});++test("default rubric replaces legacy inherited assessments and respects custom policy changes", async () => {+ const { environment, scanDirectory, rubricPath, findings, scanId } =+ await fixture();+ const { codex, calls } = recordingClassifier();+ const options = { environment, codex };+ await classifyScanDirectorySeverity(scanDirectory, options);+ // Recreate the provenance emitted by the former no-rubric inheritance path.+ for (const table of [+ "scan_severity_assessments",+ "finding_severity_assessments",+ ]) {+ await query(+ environment,+ `UPDATE ${table} SET rubric_sha256 = NULL, source = 'existing-severity', level = 'high', rubric_label = NULL, confidence = NULL`,+ );+ }+ await query(+ environment,+ "UPDATE scan_severity_classifications SET rubric_sha256 = NULL",+ );+ const legacy = await readScanSeverityClassification(+ scanDirectory,+ scanId,+ findings,+ undefined,+ environment,+ );+ expect(+ legacy?.assessments.every((a) => a.source === "existing-severity"),+ ).toBe(true);+ calls.length = 0;+ const result = await classifyScanDirectorySeverity(scanDirectory, options);+ expect(calls).toHaveLength(6);+ expect(result.rubricSha256).toMatch(/^[a-f0-9]{64}$/u);+ expect(+ result.assessments.every(+ (a) => a.source === "rubric" && a.level === "medium",+ ),+ ).toBe(true);+ calls.length = 0;+ expect(+ (await classifyScanDirectorySeverity(scanDirectory, options)).assessments,+ ).toEqual(result.assessments);+ expect(calls).toHaveLength(0);+ const custom = await classifyScanDirectorySeverity(scanDirectory, {+ ...options,+ rubricPath,+ });+ expect(calls).toHaveLength(6);+ expect(custom.rubricSha256).not.toBe(result.rubricSha256);+ calls.length = 0;+ expect(+ (await classifyScanDirectorySeverity(scanDirectory, options)).rubricSha256,+ ).toBe(result.rubricSha256);+ expect(calls).toHaveLength(6);+});++test("saved assessments bind their run count and invalidate changed or unrecorded counts", async () => {+ const { environment, scanDirectory, scanId, findings } = await fixture();+ const { codex, calls, control } = recordingClassifier();+ for (const [classificationRuns, expectedCalls] of [+ [3, 6],+ [3, 0],+ [1, 2],+ [1, 0],+ [10, 20],+ [3, 6],+ ]) {+ calls.length = 0;+ const result = await classifyScanDirectorySeverity(scanDirectory, {+ environment,+ codex,+ classificationRuns,+ });+ expect(calls).toHaveLength(expectedCalls!);+ expect(result.classificationRuns).toBe(classificationRuns);+ const saved = await readScanSeverityClassification(+ scanDirectory,+ scanId,+ findings,+ undefined,+ environment,+ );+ expect(saved?.classificationRuns).toBe(classificationRuns);+ }+ for (const table of [+ "scan_severity_classifications",+ "scan_severity_assessments",+ "finding_severity_assessments",+ ]) {+ await query(environment, `UPDATE ${table} SET classification_runs = NULL`);+ }+ expect(+ (+ await readScanSeverityClassification(+ scanDirectory,+ scanId,+ findings,+ undefined,+ environment,+ )+ )?.classificationRuns,+ ).toBeUndefined();+ calls.length = 0;+ await classifyScanDirectorySeverity(scanDirectory, { environment, codex });+ expect(calls).toHaveLength(6);+ expect(+ await query(+ environment,+ `SELECT classification_runs FROM scan_severity_classifications WHERE scan_id = '${scanId}'`,+ ),+ ).toEqual([{ classification_runs: 3 }]);+ calls.length = 0;+ control.failOn = findings[1]!.findingId;+ await expect(+ classifyScanDirectorySeverity(scanDirectory, {+ environment,+ codex,+ classificationRuns: 1,+ }),+ ).rejects.toThrow("Interrupted model call");+ await expect(+ readScanSeverityClassification(+ scanDirectory,+ scanId,+ findings,+ undefined,+ environment,+ ),+ ).rejects.toThrow("incomplete");+ calls.length = 0;+ control.failOn = "";+ await classifyScanDirectorySeverity(scanDirectory, {+ environment,+ codex,+ classificationRuns: 1,+ });+ expect(calls).toEqual([findings[1]!.findingId]);+});diff --git a/sdk/typescript/tests-ts/classify-severity-records.test.ts b/sdk/typescript/tests-ts/classify-severity-records.test.tsindex 56a8fac07..46bf803ee 100644--- a/sdk/typescript/tests-ts/classify-severity-records.test.ts+++ b/sdk/typescript/tests-ts/classify-severity-records.test.ts@@ -1,3 +1,4 @@+import { defaultSeverityRubric } from "../src/default-severity-rubric.js";import { expect, test } from "bun:test";import { createHash } from "node:crypto";import { PassThrough, Writable } from "node:stream";@@ -11,7 +12,9 @@ import { capture, dependencies } from "./cli-fixtures.js";const hash = (value: string) =>createHash("sha256").update(value, "utf8").digest("hex");-function input(): ClassifySeverityRecordsInput {+function input(): ClassifySeverityRecordsInput & {+ rubric: NonNullable<ClassifySeverityRecordsInput["rubric"]>;+} {const markdown ="# Severity\nHigh: unauthorized access to private records.\n";const findingJson =@@ -53,6 +56,9 @@ test("binds assessments to exact UTF-8 snapshots without rewriting the originalrequests.push(request);expect(request.model).toBe(original.model);expect(request.effort).toBe("high");+ expect(JSON.parse(request.prompt).rubric).toEqual([+ original.rubric.markdown,+ ]);expect(request.trustedInstructions).toContain("Do not use tools");expect(request.schema).toHaveProperty("required");expect(request.prompt).toContain(original.records[0]!.findingJson);@@ -76,7 +82,9 @@ test("binds assessments to exact UTF-8 snapshots without rewriting the original],});expect(original).toEqual(before);- expect(requests).toHaveLength(1);+ expect(requests).toHaveLength(3);+ expect(new Set(requests.map(({ requestId }) => requestId)).size).toBe(3);+ expect(new Set(requests.map(({ prompt }) => prompt)).size).toBe(1);expect(result.assessments[0]).not.toHaveProperty("findingId");});@@ -143,7 +151,7 @@ test.each([async run(request) {calls++;const result: Record<string, unknown> = answer(request);- if (calls === 1) return result;+ if (calls <= 3) return result;if (scenario === "host-failure")throw new Error("Host could not complete review");if (scenario === "wrong-id") result["findingId"] = "canonical-one";@@ -164,7 +172,7 @@ test.each([scenario === "host-failure" ? "review_failed" : "invalid_assessment",},});- expect(calls).toBe(2);+ expect(calls).toBe(4);});test("exclusion is distinct from low severity", async () => {@@ -224,6 +232,9 @@ test("cancellation rejects even when the host review does not settle", async ()test.each(["completed",+ "default-rubric",+ "one-run",+ "ten-runs","host-error","wrong-response","disconnect",@@ -272,23 +283,47 @@ test.each([throw new Error("Local scan execution is forbidden");};const result = main(- ["classify-severity", "--records"],+ [+ "classify-severity",+ "--records",+ ...(scenario === "one-run"+ ? ["--runs", "1"]+ : scenario === "ten-runs"+ ? ["--runs=10"]+ : []),+ ],output,capture().stream,deps,);+ const params: ClassifySeverityRecordsInput = input();+ if (scenario === "default-rubric") delete params.rubric;+ const completed = [+ "completed",+ "default-rubric",+ "one-run",+ "ten-runs",+ ].includes(scenario);stream.write(- `${JSON.stringify({ jsonrpc: "2.0", id: "run-one", method: "run", params: input() })}\n`,+ `${JSON.stringify({ jsonrpc: "2.0", id: "run-one", method: "run", params })}\n`,);expect(await result).toBe(- scenario === "completed" ? 0 : scenario === "host-error" ? 1 : 2,+ completed ? 0 : scenario === "host-error" ? 1 : 2,);expect(messages.filter((message) => message.method === "review.run"),- ).toHaveLength(1);+ ).toHaveLength(+ completed+ ? scenario === "one-run"+ ? 1+ : scenario === "ten-runs"+ ? 10+ : 3+ : 1,+ );const final = messages.at(-1);expect(final.id).toBe("run-one");- if (scenario === "completed")+ if (completed)expect(final.result.assessments[0]).toHaveProperty("id", "canonical-one");else if (scenario === "host-error")expect(final.result).toMatchObject({@@ -336,6 +371,7 @@ test("empty records probe version 1 without host execution", async () => {}),).toEqual({version: 1,+ classificationRuns: 3,status: "completed",rubric: { revision: data.rubric.revision, sha256: data.rubric.sha256 },assessments: [],@@ -359,3 +395,135 @@ test("immediate SDK cancellation never starts a queued host review", async () =>await expect(result).rejects.toThrow("Stopped before review");expect(calls).toBe(0);});++test("canonical records use majority voting and highest severity for a three-way tie", async () => {+ for (const [levels, expected] of [+ [["low", "high", "low"], "low"],+ [["medium", "critical", "high"], "critical"],+ ] as const) {+ let calls = 0;+ const result = await classifySeverityRecords(input(), {+ reviewRunner: {+ async run(request) {+ return { ...answer(request), level: levels[calls++] };+ },+ },+ });+ expect(calls).toBe(3);+ expect(result.assessments[0]?.level).toBe(expected);+ }+});++test.each([2, 3, 10])(+ "canonical review %s must succeed before returning any assessment",+ async (failedRun) => {+ for (const invalid of [false, true]) {+ let calls = 0;+ const data: ClassifySeverityRecordsInput = input();+ delete data.rubric;+ const result = await classifySeverityRecords(data, {+ classificationRuns: Math.max(3, failedRun),+ reviewRunner: {+ async run(request) {+ if (++calls !== failedRun) return answer(request);+ if (invalid)+ return { ...answer(request), findingId: "wrong-finding" };+ throw new Error("Host review failed");+ },+ },+ });+ expect(result).toMatchObject({+ status: "unresolved",+ assessments: [],+ error: {+ recordId: "canonical-one",+ code: invalid ? "invalid_assessment" : "review_failed",+ },+ });+ expect(calls).toBe(failedRun);+ }+ },+);++test.each([2, 3])(+ "canonical cancellation during review %s never returns a partial vote",+ async (canceledRun) => {+ const controller = new AbortController();+ let calls = 0;+ await expect(+ classifySeverityRecords(input(), {+ signal: controller.signal,+ reviewRunner: {+ async run(request) {+ if (++calls === canceledRun) controller.abort(new Error("Stopped"));+ return answer(request);+ },+ },+ }),+ ).rejects.toThrow("Stopped");+ expect(calls).toBe(canceledRun);+ },+);++test("omitted canonical rubric uses the bundled policy and returns its exact binding", async () => {+ const data: ClassifySeverityRecordsInput = input();+ delete data.rubric;+ const requests: SeverityClassificationReviewRequest[] = [];+ const result = await classifySeverityRecords(data, {+ reviewRunner: {+ async run(request) {+ requests.push(request);+ expect(JSON.parse(request.prompt).rubric).toEqual([+ defaultSeverityRubric,+ ]);+ return answer(request);+ },+ },+ });+ expect(requests).toHaveLength(3);+ expect(new Set(requests.map(({ requestId }) => requestId)).size).toBe(3);+ expect(result).toMatchObject({+ status: "completed",+ rubric: { revision: 1, sha256: hash(defaultSeverityRubric) },+ assessments: [{ level: "high" }],+ });+ expect(data.rubric).toBeUndefined();+});++test.each([1, 2, 4, 10])(+ "runs %i independent canonical reviews",+ async (classificationRuns) => {+ let calls = 0;+ const result = await classifySeverityRecords(input(), {+ classificationRuns,+ reviewRunner: {+ async run(request) {+ calls++;+ return answer(request);+ },+ },+ });+ expect(calls).toBe(classificationRuns);+ expect(result.classificationRuns).toBe(classificationRuns);+ expect(result.status).toBe("completed");+ },+);++test.each([0, -1, 11, 1.5, NaN, Infinity])(+ "rejects invalid canonical run count %s before host work",+ async (classificationRuns) => {+ let calls = 0;+ await expect(+ classifySeverityRecords(input(), {+ classificationRuns,+ reviewRunner: {+ async run(request) {+ calls++;+ return answer(request);+ },+ },+ }),+ ).rejects.toThrow();+ expect(calls).toBe(0);+ },+);diff --git a/sdk/typescript/tests-ts/classify-severity.test.ts b/sdk/typescript/tests-ts/classify-severity.test.tsindex b972a4aa1..e0488fcdc 100644--- a/sdk/typescript/tests-ts/classify-severity.test.ts+++ b/sdk/typescript/tests-ts/classify-severity.test.ts@@ -1,3 +1,5 @@+import { defaultSeverityRubric } from "../src/default-severity-rubric.js";+import { workflowDigest } from "../src/finding-workflow.js";import { modelResponseText } from "./support/model-response-text.js";import { writeFile } from "node:fs/promises";import { join } from "node:path";@@ -53,7 +55,13 @@ function fakeCodex(response: unknown) {return {async run(prompt, turn) {calls.push({ prompt, thread, turn });- return { finalResponse: modelResponseText(response) };+ return {+ finalResponse: modelResponseText(+ typeof response === "function"+ ? response(calls.length)+ : response,+ ),+ };},};},@@ -61,19 +69,23 @@ function fakeCodex(response: unknown) {return { codex, calls };}-test("without a rubric reuses severity without authentication or a model call", async () => {- const { codex, calls } = fakeCodex("must not run");+test("without a rubric runs three independent reviews with the built-in policy", async () => {+ const { codex, calls } = fakeCodex(assessed);const original = structuredClone(finding);const result = await classifySeverity([finding], { codex, environment: {} });- expect(calls).toHaveLength(0);- expect(result.rubricSha256).toBeNull();+ expect(calls).toHaveLength(3);+ expect(new Set(calls.map((call) => call.thread)).size).toBe(3);+ expect(JSON.parse(calls[0]!.prompt.split("\n\n").at(-1)!).rubric).toEqual([+ defaultSeverityRubric,+ ]);+ expect(result.rubricSha256).toBe(workflowDigest([defaultSeverityRubric]));expect(result.assessments[0]).toMatchObject({findingId: finding.findingId,occurrenceId: finding.occurrenceId,decision: "assessed",- level: "high",- source: "existing-severity",- rationale: "Original assessment",+ level: "medium",+ source: "rubric",+ rationale: assessed.rationale,});expect(finding).toEqual(original);expect(@@ -104,7 +116,13 @@ test("supplies complete evidence and separate policy/context to a restricted str});expect(result.rubricSha256).toMatch(/^[a-f0-9]{64}$/u);expect(result.knowledgeBaseSha256).toMatch(/^[a-f0-9]{64}$/u);- expect(calls).toHaveLength(1);+ expect(calls).toHaveLength(3);+ expect(new Set(calls.map((call) => call.thread)).size).toBe(3);+ expect(new Set(calls.map((call) => call.prompt)).size).toBe(1);+ expect(JSON.parse(calls[0]!.prompt.split("\n\n").at(-1)!).rubric).toEqual([+ "Assign MEDIUM to bounded unauthorized metadata reads.",+ ]);+ expect(calls[0]!.prompt).not.toContain(defaultSeverityRubric);expect(calls[0]!.prompt).toContain(String(finding["evidence"]));expect(calls[0]!.prompt).toContain("The counters contain no customer data.");expect(calls[0]!.thread).toMatchObject({@@ -151,19 +169,15 @@ test("represents policy exclusions independently from Low", async () => {expect(validateSeverityClassification(result, [finding])).toEqual(result);});-test("classifies imported reports without severity when a rubric is supplied", async () => {+test("classifies imported reports without severity using the default rubric", async () => {const {severity: _severity,occurrenceId: _occurrenceId,...imported} = finding;- await expect(classifySeverity([imported])).rejects.toThrow("supply a rubric");- const rubricPath = await document(- "Classify bounded metadata reads as medium.",- );const { codex } = fakeCodex(assessed);expect(- (await classifySeverity([imported], { rubricPath, codex })).assessments[0]!+ (await classifySeverity([imported], { codex })).assessments[0]!.occurrenceId,).toBeNull();});@@ -230,3 +244,104 @@ test("cancellation and invalid inputs cannot produce a successful classification);expect((await classifySeverity([])).assessments).toEqual([]);});++test("votes independently for each imported report and retains the winning answer", async () => {+ const rubricPath = await document("Classify using the supplied evidence.");+ const second = { ...finding, findingId: "second-finding" };+ const responses = [+ { ...assessed, level: "low", rationale: "First low answer" },+ { ...assessed, level: "high" },+ { ...assessed, level: "low", rationale: "Second low answer" },+ ...["medium", "critical", "high"].map((level) => ({+ ...assessed,+ findingId: second.findingId,+ level,+ })),+ ];+ const { codex, calls } = fakeCodex((call: number) => responses[call - 1]);+ const result = await classifySeverity([finding, second], {+ rubricPath,+ codex,+ });+ expect(calls).toHaveLength(6);+ expect(result.assessments).toMatchObject([+ { level: "low", rationale: "First low answer" },+ { level: "critical", findingId: second.findingId },+ ]);+ expect(+ calls.map(+ ({ prompt }) =>+ JSON.parse(prompt.split("\n\n").at(-1)!).finding.findingId,+ ),+ ).toEqual([+ finding.findingId,+ finding.findingId,+ finding.findingId,+ second.findingId,+ second.findingId,+ second.findingId,+ ]);+});++test.each([2, 3, 10])(+ "does not return a partial vote when review %s fails or is invalid",+ async (failedRun) => {+ for (const invalid of [false, true]) {+ const { codex, calls } = fakeCodex((call: number) => {+ if (call !== failedRun) return assessed;+ if (invalid) return { ...assessed, findingId: "wrong-finding" };+ throw new Error("Review failed");+ });+ await expect(+ classifySeverity([finding], {+ codex,+ classificationRuns: Math.max(3, failedRun),+ }),+ ).rejects.toThrow(invalid ? "invalid assessment" : "Review failed");+ expect(calls).toHaveLength(failedRun);+ }+ },+);++test.each([1, 2, 3])(+ "cancellation after review %s stops classification",+ async (canceledRun) => {+ const controller = new AbortController();+ const { codex, calls } = fakeCodex((call: number) => {+ if (call === canceledRun) controller.abort(new Error("Stopped"));+ return assessed;+ });+ await expect(+ classifySeverity([finding], {+ codex,+ signal: controller.signal,+ }),+ ).rejects.toThrow("Stopped");+ expect(calls).toHaveLength(canceledRun);+ },+);++test.each([1, 2, 4, 10])(+ "runs %i independent local reviews",+ async (classificationRuns) => {+ const { codex, calls } = fakeCodex(assessed);+ const result = await classifySeverity([finding], {+ codex,+ classificationRuns,+ });+ expect(calls).toHaveLength(classificationRuns);+ expect(result.classificationRuns).toBe(classificationRuns);+ expect(result.assessments[0]!.level).toBe("medium");+ },+);++test.each([0, -1, 11, 1.5, NaN, Infinity])(+ "rejects invalid local run count %s before model work",+ async (classificationRuns) => {+ const { codex, calls } = fakeCodex(assessed);+ await expect(+ classifySeverity([finding], { codex, classificationRuns }),+ ).rejects.toThrow();+ expect(calls).toHaveLength(0);+ },+);diff --git a/sdk/typescript/tests-ts/cli-classify-severity.test.ts b/sdk/typescript/tests-ts/cli-classify-severity.test.tsindex 262064afb..0026574ec 100644--- a/sdk/typescript/tests-ts/cli-classify-severity.test.ts+++ b/sdk/typescript/tests-ts/cli-classify-severity.test.ts@@ -67,7 +67,7 @@ test.each([},);-test("classify-severity accepts external scan directories and defaults to existing severity", async () => {+test("classify-severity accepts external scan directories and uses the default rubric when none is supplied", async () => {const deps = dependencies();let called = false;deps.classifyScanDirectorySeverity = async (directory, options, surface) => {@@ -75,6 +75,7 @@ test("classify-severity accepts external scan directories and defaults to existiexpect(directory).toBe(resolve(deps.currentDirectory(), "saved scan"));expect(options!.rubricPath).toBeUndefined();expect(options!.reprocess).toBe(false);+ expect(options!.classificationRuns).toBe(3);expect(options!.findingIds).toBeUndefined();expect(surface).toBe("cli");return result;@@ -195,3 +196,45 @@ test("publication forwards selected finding IDs only to Linear", async () => {),).toBe(2);});++test.each([1, 2, 10])("forwards --runs %i", async (classificationRuns) => {+ const deps = dependencies();+ const classify = mock(async (_scan: string, options: any) => {+ expect(options.classificationRuns).toBe(classificationRuns);+ return result;+ });+ deps.classifyScanSeverity = classify;+ expect(+ await runCapturedCli(+ main,+ [+ "classify-severity",+ "--scan",+ "latest",+ "--runs",+ String(classificationRuns),+ ],+ deps,+ ),+ ).toBe(0);+ expect(classify).toHaveBeenCalledTimes(1);+});++test.each(["0", "-1", "11", "1.5", "NaN", "Infinity", "invalid"])(+ "rejects --runs %s in either input mode",+ async (value) => {+ const deps = dependencies();+ const classify = mock(async () => result);+ deps.classifyScanSeverity = classify;+ for (const mode of [["--scan", "latest"], ["--records"]]) {+ expect(+ await runCapturedCli(+ main,+ ["classify-severity", ...mode, "--runs", value],+ deps,+ ),+ ).toBe(2);+ }+ expect(classify).toHaveBeenCalledTimes(0);+ },+);diff --git a/sdk/typescript/tests-ts/severity-review.test.ts b/sdk/typescript/tests-ts/severity-review.test.tsnew file mode 100644index 000000000..c7f3496f6--- /dev/null+++ b/sdk/typescript/tests-ts/severity-review.test.ts@@ -0,0 +1,77 @@+import { expect, test } from "bun:test";+import {+ reviewSeverity,+ type SeverityDecision,+} from "../src/severity-review.js";++// Listed from lowest to highest independently of the runtime ordering.+const levels = [+ null,+ "informational",+ "low",+ "medium",+ "high",+ "critical",+] as const;++test("three independent reviews vote by normalized severity, with deterministic ties", async () => {+ for (const first of levels) {+ for (const second of levels) {+ for (const third of levels) {+ const votes = [first, second, third] as const;+ const answers: SeverityDecision[] = votes.map((level, index) => ({+ findingId: "example-finding",+ decision: level === null ? "excluded" : "assessed",+ level,+ rubricLabel: level === null ? null : `${level}-${index}`,+ rationale: `Review ${index + 1}`,+ confidence: index === 0 ? "low" : "high",+ reviewTrigger: `Missing fact ${index + 1}`,+ }));+ let calls = 0;+ const result = await reviewSeverity(async () => answers[calls++]!);+ const expected =+ first === second || first === third+ ? first+ : second === third+ ? second+ : levels[+ Math.max(...votes.map((level) => levels.indexOf(level)))+ ];+ expect(calls).toBe(3);+ expect(result).toBe(answers.find(({ level }) => level === expected)!);+ }+ }+ }+});++test.each([+ [["low"], "low"],+ [["low", "high"], "high"],+ [["low", "low", "high", "high"], "high"],+ [["critical", "low", "low", "high", "high", "informational"], "high"],+ [[null, null, "low", "low"], "low"],+ [+ ["low", "low", "low", "low", "low", "high", "high", "high", "high", "high"],+ "high",+ ],+] as [SeverityDecision["level"][], SeverityDecision["level"]][])(+ "votes across %j",+ async (votes, expected) => {+ let calls = 0;+ const result = await reviewSeverity(async () => {+ const level = votes[calls++]!;+ return {+ findingId: "example",+ decision: level === null ? "excluded" : "assessed",+ level,+ rubricLabel: level,+ rationale: "Synthetic review",+ confidence: null,+ reviewTrigger: null,+ };+ }, votes.length);+ expect(calls).toBe(votes.length);+ expect(result.level).toBe(expected);+ },+);