Skip to content

OpenAI's Codex Security CLI and TypeScript SDK for finding, validating, and fixing security vulnerabilities. npm: https://www.npmjs.com/package/@openai/codex-security

fix!: classify severity with a default rubric and configurable reviews (#1765)

Kyle Brown8c00f36cda71
diff --git a/plugins/codex-security/mcp-app/src/workbench/severity.ts b/plugins/codex-security/mcp-app/src/workbench/severity.ts
index d3f53e4f8..ca0c18fa8 100644
--- a/plugins/codex-security/mcp-app/src/workbench/severity.ts
+++ b/plugins/codex-security/mcp-app/src/workbench/severity.ts
@@ -5,6 +5,7 @@ import type { Finding } from "./findings";
import { transaction } from "./transaction";
const fields = {
+ classificationRuns: "classification_runs",
findingId: "finding_id",
occurrenceId: "occurrence_id",
inputSha256: "input_sha256",
@@ -20,7 +21,10 @@ const fields = {
reviewTrigger: "review_trigger",
} as const;
-type Assessment = Record<keyof typeof fields, string | null>;
+type Assessment = Record<
+ Exclude<keyof typeof fields, "classificationRuns">,
+ string | null
+> & { classificationRuns?: number | null };
export type SeverityCheckpoint =
| {
@@ -30,6 +34,7 @@ export type SeverityCheckpoint =
assessedAt: string;
rubricSha256: string | null;
knowledgeBaseSha256: string | null;
+ classificationRuns?: number;
}
| {
action: "save";
@@ -50,7 +55,12 @@ export function assessments(
: "scan_severity_assessments";
const scope = scanId === undefined ? "" : "WHERE assessment.scan_id = ?";
// JSON preserves embedded NULs across Node 22 SQLite TEXT results.
+ const hasRunCount = database
+ .prepare(`PRAGMA table_info(${table})`)
+ .all()
+ .some((column) => column.name === "classification_runs");
const projection = Object.entries(fields)
+ .filter(([key]) => key !== "classificationRuns" || hasRunCount)
.map(([key, column]) => `'${key}', assessment.${column}`)
.join(", ");
// Keep selected IDs first so each assessment is a primary-key lookup.
@@ -84,13 +94,14 @@ export function severityCheckpoint(
database
.prepare(
`INSERT INTO scan_severity_classifications (
- scan_id, finding_ids_json, assessed_at, rubric_sha256, knowledge_base_sha256
- ) VALUES (?, ?, ?, ?, ?)
+ scan_id, finding_ids_json, assessed_at, rubric_sha256, knowledge_base_sha256, classification_runs
+ ) VALUES (?, ?, ?, ?, ?, ?)
ON CONFLICT(scan_id) DO UPDATE SET
finding_ids_json = excluded.finding_ids_json,
assessed_at = excluded.assessed_at,
rubric_sha256 = excluded.rubric_sha256,
- knowledge_base_sha256 = excluded.knowledge_base_sha256`,
+ knowledge_base_sha256 = excluded.knowledge_base_sha256,
+ classification_runs = excluded.classification_runs`,
)
.run(
payload.scanId,
@@ -98,6 +109,7 @@ export function severityCheckpoint(
payload.assessedAt,
payload.rubricSha256,
payload.knowledgeBaseSha256,
+ payload.classificationRuns ?? null,
);
// Cache hits are not saved again by the classifier. Keep each scan's own assessments.
database
@@ -121,10 +133,15 @@ export function severityCheckpoint(
const values = Object.fromEntries(
Object.entries(fields).map(([key, column]) => [
column,
- assessment[key as keyof Assessment],
+ assessment[key as keyof Assessment] ?? null,
]),
);
- requireSqliteText([finding.findingId, ...Object.values(values)]);
+ requireSqliteText([
+ finding.findingId,
+ ...Object.values(values).filter(
+ (value): value is string => typeof value === "string",
+ ),
+ ]);
// External scan directories may not have been indexed on this machine.
if (
!database
@@ -191,11 +208,16 @@ export function readSeverityClassification(
.get()
)
return {};
+ const hasRunCount = database
+ .prepare("PRAGMA table_info(scan_severity_classifications)")
+ .all()
+ .some((column) => column.name === "classification_runs");
const row = database
.prepare(
`SELECT json_object(
'findingIds', json(finding_ids_json), 'assessedAt', assessed_at,
'rubricSha256', rubric_sha256, 'knowledgeBaseSha256', knowledge_base_sha256
+ ${hasRunCount ? ", 'classificationRuns', classification_runs" : ""}
) AS value FROM scan_severity_classifications WHERE scan_id = ?`,
)
.get(scanId);
diff --git a/plugins/codex-security/mcp-app/tests/test_workbench_severity.ts b/plugins/codex-security/mcp-app/tests/test_workbench_severity.ts
index e1ee0cb36..af19fdf11 100644
--- a/plugins/codex-security/mcp-app/tests/test_workbench_severity.ts
+++ b/plugins/codex-security/mcp-app/tests/test_workbench_severity.ts
@@ -43,6 +43,7 @@ function save(
evidence: { count: 18446744073709551617n },
},
assessment: {
+ classificationRuns: null,
findingId,
occurrenceId: `occurrence-${findingId}`,
inputSha256: "evidence-digest",
@@ -77,6 +78,7 @@ function begin(
test("severity checkpoints retain external findings and each scan's cached assessments", (t) => {
const database = open(t);
const payload = save("finding\0suffix", "scan-a");
+ payload.assessment.classificationRuns = 10;
for (const field of [
"occurrenceId",
"inputSha256",
diff --git a/plugins/codex-security/shared/workbench-migrations.json b/plugins/codex-security/shared/workbench-migrations.json
index 0a18f1649..1e6a725b0 100644
--- a/plugins/codex-security/shared/workbench-migrations.json
+++ b/plugins/codex-security/shared/workbench-migrations.json
@@ -425,5 +425,14 @@
"CREATE TRIGGER record_deep_scan_worker_thread_insert\nAFTER INSERT ON deep_scan_workers\nWHEN NEW.sdk_thread_id IS NOT NULL\nBEGIN\n INSERT OR IGNORE INTO deep_scan_worker_threads (scan_id, worker_id, sdk_thread_id)\n VALUES (NEW.scan_id, NEW.id, NEW.sdk_thread_id);\nEND;",
"CREATE TRIGGER record_deep_scan_worker_thread_update\nAFTER UPDATE OF sdk_thread_id ON deep_scan_workers\nWHEN NEW.sdk_thread_id IS NOT NULL\nBEGIN\n INSERT OR IGNORE INTO deep_scan_worker_threads (scan_id, worker_id, sdk_thread_id)\n VALUES (NEW.scan_id, NEW.id, NEW.sdk_thread_id);\nEND;"
]
+ },
+ {
+ "version": 52,
+ "name": "record severity classification review counts",
+ "statements": [
+ "ALTER TABLE finding_severity_assessments ADD COLUMN classification_runs INTEGER CHECK (classification_runs BETWEEN 1 AND 10);",
+ "ALTER TABLE scan_severity_assessments ADD COLUMN classification_runs INTEGER CHECK (classification_runs BETWEEN 1 AND 10);",
+ "ALTER TABLE scan_severity_classifications ADD COLUMN classification_runs INTEGER CHECK (classification_runs BETWEEN 1 AND 10);"
+ ]
}
]
diff --git a/plugins/codex-security/tests/test_workbench_setup_and_migrations.py b/plugins/codex-security/tests/test_workbench_setup_and_migrations.py
index 59b149f4d..f1f7ec12d 100644
--- a/plugins/codex-security/tests/test_workbench_setup_and_migrations.py
+++ b/plugins/codex-security/tests/test_workbench_setup_and_migrations.py
@@ -93,6 +93,7 @@ EXPECTED_MIGRATIONS = [
(49, "index finding issue receipt ownership"),
(50, "persist authorized source excerpt scopes"),
(51, "preserve deep scan worker thread history"),
+ (52, "record severity classification review counts"),
]
diff --git a/sdk/typescript/README.md b/sdk/typescript/README.md
index acaa4ed04..84d15ab8b 100644
--- a/sdk/typescript/README.md
+++ b/sdk/typescript/README.md
@@ -612,8 +612,11 @@ and [Azure Pipelines](https://github.com/openai/codex-security/blob/main/example
### Classify finding severity
-Use `classify-severity` with your own rubric to assess saved findings without
-changing their original severity or evidence. The SDK exposes `classifySeverity()`,
+Use `classify-severity` to assess saved findings with the built-in severity rubric
+or an explicit custom rubric, without changing their original severity or evidence.
+Each finding receives three independent reviews by default; the most common severity wins,
+with the highest severity breaking ties among the top vote counts.
+Use `--runs N` to request 1–10 reviews (default: 3). The SDK exposes `classifySeverity()`,
`classifyScanSeverity()`, and `classifyScanDirectorySeverity()`. See
[severity classification](docs/cli.md#classify-finding-severity) for selection,
checkpointing, and publication behavior.
diff --git a/sdk/typescript/docs/classify-severity-records.md b/sdk/typescript/docs/classify-severity-records.md
index 07abf11d3..a1d2cb29c 100644
--- a/sdk/typescript/docs/classify-severity-records.md
+++ b/sdk/typescript/docs/classify-severity-records.md
@@ -3,7 +3,7 @@
`codex-security classify-severity --records` accepts canonical reports without a
repository, saved scan, synthetic provenance, local Codex login, or filesystem
state. It uses the same bidirectional JSON-RPC transport as [record deduplication](dedupe-records.md).
-Use `--records` alone; saved-scan classification remains unchanged.
+Use `--records` alone or with `--runs N`; model execution remains host-provided.
The SDK exports `classifySeverityRecords`, `classifySeverityRecordsInputSchema`,
and their input, result, and review-request types. The protocol and input have
@@ -25,6 +25,15 @@ Send one newline-delimited JSON-RPC request with `method: "run"` and these param
}
```
+The `rubric` field is optional. Omit it to classify with the
+[built-in severity rubric](../src/default-severity-rubric.ts), using the same configured
+independent host reviews and usage accounting. The result echoes its built-in
+revision (currently `1`) and exact Markdown SHA-256. Supplying a custom rubric
+replaces the default completely and preserves the supplied revision/hash binding.
+Do not reuse cached results across differing rubric hashes; preserve the default
+snapshot from the review prompt alongside its returned binding. Default revisions
+are local to the bundled policy, not workspace rubric revisions.
+
`findingJson` is the exact JSON text of an object with nonblank `findingId`,
`title`, and `summary`. `findingId` must equal the canonical `id`. Optional
`severity.level` uses `critical`, `high`, `medium`, `low`, or `informational`.
@@ -36,7 +45,7 @@ normalizing newlines, whitespace, Unicode, object key order, or JSON numbers.
Compute `inputSha256` from `findingJson`, not from its parsed/reserialized object.
For example, Node uses `createHash("sha256").update(text, "utf8").digest("hex")`;
Python uses `hashlib.sha256(text.encode("utf-8")).hexdigest()`.
-The CLI recomputes both hashes before requesting any review. Rubric revisions are
+The CLI recomputes supplied hashes before requesting any review. Rubric revisions are
positive safe integers; record revisions are nonnegative safe integers. Duplicate
canonical IDs and invalid records reject the entire input before host work.
The implementation snapshots caller input before awaiting model execution and
@@ -45,8 +54,10 @@ numeric evidence that JavaScript cannot represent exactly.
## Host execution contract
-For each record the CLI emits `method: "review.run"`, with a unique request ID
-and params containing `requestId`, `stage: "severity-classification"`, explicit
+The `classificationRuns` SDK option or CLI `--runs N` flag accepts
+an integer from 1 through 10 (default: 3), validated before any host work.
+For each record the CLI emits that many sequential `method: "review.run"` requests,
+each with a unique request ID and params containing `requestId`, `stage: "severity-classification"`, explicit
`model` and `effort`, `trustedInstructions`, untrusted report/rubric `prompt`, and
an output JSON `schema`. The host must authorize the requested model and effort,
create a fresh independent review context, preserve the instruction/data boundary,
@@ -61,7 +72,14 @@ object. Do not wrap the assessment in a string or an array. An assessment contai
`rubricLabel`, `rationale`, separate `confidence`, and nullable `reviewTrigger`.
Excluded assessments require null level and label; assessed ones require both.
-The host owns model budgets, transport limits, timeouts, and session cancellation.
+The reviews receive identical inputs and do not see earlier answers. The
+most common normalized severity wins; the highest among tied top counts wins
+(`critical` > `high` > `medium` > `low` > `informational`). Exclusion is a separate
+vote: exclusions can win by count, but lose ties to assessed severities. The first answer matching the winner supplies all assessment fields.
+
+The host owns model budgets, usage accounting, transport limits, timeouts, and
+session cancellation. Account for every review request, including work preceding
+a later failure; classification results do not aggregate or report token usage.
Use the transport's `cancel` notification with the active run ID and also stop
any remote execution. EOF or a broken pipe means disconnection, not successful
completion. No review is retried automatically, including after an ambiguous
@@ -70,20 +88,20 @@ host execution error.
## Results and persistence
A completed result contains `version: 1`, `status: "completed"`, the rubric's
-`revision` and `sha256`, and `assessments`. Each assessment replaces model-provided
+`revision` and `sha256`, `classificationRuns`, and `assessments`. Each assessment replaces model-provided
`findingId` with the verified canonical `id`, `revision`, and `inputSha256`.
The original report and severity are never rewritten.
Invalid or mismatched model output and host failures return `status: "unresolved"`,
an empty `assessments` array, and an error with `recordId` and stable code
-`invalid_assessment` or `review_failed`. Earlier assessments in that invocation
-are discarded. Invalid input, protocol errors, and cancellation use JSON-RPC error
-responses. Exit codes are 0 for completion, 1 for unresolved review, and 2 for
+`invalid_assessment` or `review_failed`. All requested reviews must succeed; partial
+votes and earlier assessments in that invocation are discarded. Invalid input,
+protocol errors, and cancellation use JSON-RPC error responses. Exit codes are 0 for completion, 1 for unresolved review, and 2 for
input/protocol failure (130/143 for SIGINT/SIGTERM).
Prefer one canonical record per workflow invocation when the host needs independent
retries and partial success. The host should retain immutable rubric/report
-snapshots, compare every echoed binding, and conditionally persist only computed
+snapshots, compare every echoed binding including `classificationRuns`, and conditionally persist only computed
prioritization fields if the expected canonical version and rubric still apply.
Keep manual edits and original severity intact. Repeating the same inputs binds
results to the same hashes; it does not promise deterministic model answers.
diff --git a/sdk/typescript/docs/cli.md b/sdk/typescript/docs/cli.md
index 75b0ca7e4..799f22a0f 100644
--- a/sdk/typescript/docs/cli.md
+++ b/sdk/typescript/docs/cli.md
@@ -1283,16 +1283,31 @@ codex-security classify-severity --scan latest --rubric /path/to/policy.md --jso
codex-security classify-severity --scan-dir /path/to/completed-scan --json
```
-For repository-free canonical records, use `classify-severity --records` alone.
+For repository-free canonical records, use `classify-severity --records` (optionally with `--runs N`).
The [records protocol](classify-severity-records.md) accepts immutable Markdown and
report snapshots and delegates model execution to the host.
For saved findings, use exactly one of `--scan` (ID, prefix, or `latest`) and `--scan-dir`.
-Omitting `--rubric` inherits original severity without model calls. Rubrics and
-repeatable `--knowledge-base` context accept text, PDF, DOCX, and directories.
-Each finding is classified in a separate read-only turn using its report and
-supplied context, without tools or source inspection. `--model` and `--effort`
-override Codex's configured model and the default medium effort.
+Omitting `--rubric` runs classification with the [built-in severity rubric](../src/default-severity-rubric.ts).
+It evaluates attacker access, boundary crossing, additional harm, and evidence for
+Critical, High, Medium, Low, and Informational findings. Confidence remains separate
+from severity; missing proof alone does not justify a downgrade. An explicit
+`--rubric` replaces the default policy completely. Rubrics and repeatable
+`--knowledge-base` context accept text, PDF, DOCX, and directories.
+Each finding is classified three times by default. `--runs N` accepts
+an integer from 1 through 10; SDK options use `classificationRuns` with the same
+default and bounds. Each review runs in an independent read-only turn using the
+same report, rubric, and supplied context, without tools or source inspection.
+The most common normalized severity wins; the highest among tied top counts wins
+(`critical` > `high` > `medium` > `low` > `informational`). Exclusion is a separate
+vote: exclusions can win by count, but lose ties to assessed severities. The first answer matching the winner supplies its label, rationale,
+confidence, and review trigger. All requested reviews must succeed; failures or invalid
+answers fail that finding without saving a partial vote. This uses the requested number of model
+calls per uncached finding (three by default), including when no custom rubric is supplied, and
+requires the same Codex authentication and model access as custom classification.
+Codex retains usage for each independent turn. There is no fallback to scanner
+severity if classification fails.
+`--model` and `--effort` override Codex's configured model and the default medium effort.
Assessments contain `decision` (`assessed` or `excluded`), normalized `level`,
the original `rubricLabel`, rationale, confidence, and a `reviewTrigger` for
@@ -1300,10 +1315,17 @@ missing facts that could change the result. Exclusions have a null level.
Finding/occurrence IDs and input hashes bind each assessment to its evidence.
Successful assessments checkpoint immediately in local SQLite. Retries reuse
-matching evidence, rubric, and context hashes; failed/canceled runs keep completed
-work. Use `--reprocess` to replace assessments, including when changing only model
-or effort. A successful run also writes `severity-classification.json`, but
-publication reads the database. Original findings and sealed artifacts remain
+matching evidence, rubric, context hashes, and classification run count; failed/canceled runs keep completed
+work. Default-rubric assessments use `source: "rubric"` and a content hash; former
+no-rubric inherited assessments have no rubric hash and are automatically
+reclassified on the next classification run. Changes to the bundled policy or
+switching between default and custom policies also invalidate matching checkpoints.
+The result and checkpoint record `classificationRuns`; changing the count, or
+reusing an older checkpoint with no recorded count, triggers fresh classification.
+Older results remain readable; publication does not silently run a model.
+Use `--reprocess` to replace assessments, including when changing only model
+or effort. A successful
+run also writes `severity-classification.json`, but publication reads the database. Original findings and sealed artifacts remain
unchanged.
Repeat `--finding-id ID` for a subset from the selected scan. Dedupe's
@@ -1374,8 +1396,9 @@ await publishScan(scanDirectory, {
```
`classifySeverity()` accepts `findingId`, `title`, `summary`, and available
-evidence/metadata. Imported reports may lack severity or occurrence IDs; missing
-severity requires a rubric. Scan wrappers persist checkpoints (including for
+evidence/metadata. Imported reports may lack severity or occurrence IDs; omitting
+`rubricPath` uses the built-in policy even when a scanner severity is present.
+Scan wrappers persist checkpoints (including for
external directories), accept `reprocess: true`, and treat `findingIds: []` as an
empty selection. Use the same `CODEX_SECURITY_STATE_DIR` for classification and
publication. Older JSON-only assessments need reclassification once. All
diff --git a/sdk/typescript/scripts/package-dist-files.mjs b/sdk/typescript/scripts/package-dist-files.mjs
index 897be9037..e57af79bd 100644
--- a/sdk/typescript/scripts/package-dist-files.mjs
+++ b/sdk/typescript/scripts/package-dist-files.mjs
@@ -20,6 +20,7 @@ const distModules = [
"classify-scan-severity",
"severity-store",
"severity-review",
+ "default-severity-rubric",
"cloud-publish",
"codex-prompt",
"codex-sdk-environment",
diff --git a/sdk/typescript/src/classify-severity-records.ts b/sdk/typescript/src/classify-severity-records.ts
index 335d29cd5..3361f8f94 100644
--- a/sdk/typescript/src/classify-severity-records.ts
+++ b/sdk/typescript/src/classify-severity-records.ts
@@ -1,9 +1,12 @@
import { SCAN_SEVERITIES } from "./scan-settings.js";
+import { defaultSeverityRubric } from "./default-severity-rubric.js";
import { createHash, randomUUID } from "node:crypto";
import { z } from "zod";
import { abortable } from "./targets.js";
import {
parseSeverityDecision,
+ reviewSeverity,
+ classificationRunsSchema,
severityDecisionSchema,
severityReviewInstructions,
type SeverityDecision,
@@ -41,7 +44,8 @@ export const classifySeverityRecordsInputSchema = z
markdown: text,
sha256: digest,
})
- .strict(),
+ .strict()
+ .optional(),
model: text,
effort: z.enum([
"minimal",
@@ -80,6 +84,8 @@ export interface SeverityClassificationReviewRequest {
}
export interface ClassifySeverityRecordsOptions {
+ /** Independent reviews per record: integer 1–10, default 3. */
+ classificationRuns?: number;
reviewRunner: {
run(
request: SeverityClassificationReviewRequest,
@@ -99,6 +105,7 @@ export type CanonicalSeverityAssessment = Omit<
};
interface RecordsResultBinding {
+ classificationRuns: number;
version: 1;
rubric: { revision: number; sha256: string };
}
@@ -130,8 +137,16 @@ export async function classifySeverityRecords(
options: ClassifySeverityRecordsOptions,
): Promise<ClassifySeverityRecordsResult> {
options.signal?.throwIfAborted();
+ const classificationRuns = classificationRunsSchema
+ .default(3)
+ .parse(options.classificationRuns);
const data = classifySeverityRecordsInputSchema.parse(structuredClone(input));
- if (sha256(data.rubric.markdown) !== data.rubric.sha256)
+ const rubric = data.rubric ?? {
+ revision: 1,
+ markdown: defaultSeverityRubric,
+ sha256: sha256(defaultSeverityRubric),
+ };
+ if (sha256(rubric.markdown) !== rubric.sha256)
throw new Error("Rubric SHA-256 does not match its exact UTF-8 Markdown.");
const ids = new Set<string>();
for (const record of data.records) {
@@ -147,49 +162,40 @@ export async function classifySeverityRecords(
);
}
const binding: RecordsResultBinding = {
+ classificationRuns,
version: 1,
- rubric: { revision: data.rubric.revision, sha256: data.rubric.sha256 },
+ rubric: { revision: rubric.revision, sha256: rubric.sha256 },
};
const assessments: CanonicalSeverityAssessment[] = [];
for (const record of data.records) {
options.signal?.throwIfAborted();
- let response: unknown;
+ let code: "review_failed" | "invalid_assessment" = "review_failed";
try {
- response = await abortable(() => {
- options.signal?.throwIfAborted();
- return options.reviewRunner.run(
- {
- requestId: randomUUID(),
- stage: "severity-classification",
- model: data.model,
- effort: data.effort,
- trustedInstructions: severityReviewInstructions,
- prompt: `{"rubric":${JSON.stringify([data.rubric.markdown])},"knowledgeBase":null,"finding":${record.findingJson}}`,
- schema: z.toJSONSchema(severityDecisionSchema),
- },
- { signal: options.signal },
- );
- }, options.signal);
- } catch (error) {
- options.signal?.throwIfAborted();
- return {
- ...binding,
- status: "unresolved",
- assessments: [],
- error: {
- recordId: record.id,
- code: "review_failed",
- message:
- error instanceof Error ? error.message : "Host review failed.",
+ const { findingId: _findingId, ...decision } = await reviewSeverity(
+ async () => {
+ code = "review_failed";
+ const response = await abortable(() => {
+ options.signal?.throwIfAborted();
+ return options.reviewRunner.run(
+ {
+ requestId: randomUUID(),
+ stage: "severity-classification",
+ model: data.model,
+ effort: data.effort,
+ trustedInstructions: severityReviewInstructions,
+ prompt: `{"rubric":${JSON.stringify([rubric.markdown])},"knowledgeBase":null,"finding":${record.findingJson}}`,
+ schema: z.toJSONSchema(severityDecisionSchema),
+ },
+ { signal: options.signal },
+ );
+ }, options.signal);
+ options.signal?.throwIfAborted();
+ code = "invalid_assessment";
+ return parseSeverityDecision(response, record.id);
},
- };
- }
- options.signal?.throwIfAborted();
- try {
- const { findingId: _findingId, ...decision } = parseSeverityDecision(
- response,
- record.id,
+ classificationRuns,
);
+ options.signal?.throwIfAborted();
assessments.push({
...decision,
id: record.id,
@@ -197,15 +203,20 @@ export async function classifySeverityRecords(
inputSha256: record.inputSha256,
});
} catch (error) {
+ options.signal?.throwIfAborted();
return {
...binding,
status: "unresolved",
assessments: [],
error: {
recordId: record.id,
- code: "invalid_assessment",
+ code,
message:
- error instanceof Error ? error.message : "Invalid assessment.",
+ error instanceof Error
+ ? error.message
+ : code === "review_failed"
+ ? "Host review failed."
+ : "Invalid assessment.",
},
};
}
diff --git a/sdk/typescript/src/classify-severity.ts b/sdk/typescript/src/classify-severity.ts
index 8fbf01e0f..2e65ffdab 100644
--- a/sdk/typescript/src/classify-severity.ts
+++ b/sdk/typescript/src/classify-severity.ts
@@ -1,4 +1,4 @@
-import { SCAN_SEVERITIES } from "./scan-settings.js";
+import { defaultSeverityRubric } from "./default-severity-rubric.js";
import { findingEntry } from "./value.js";
import { z } from "zod";
import { CodexSecurityError } from "./errors.js";
@@ -13,6 +13,8 @@ import {
severityDecisionSchema as decisionSchema,
severityReviewInstructions,
parseSeverityDecision,
+ reviewSeverity,
+ classificationRunsSchema,
} from "./severity-review.js";
import { CODEX_SECURITY_THREAD_SOURCES } from "./thread-source.js";
@@ -33,8 +35,10 @@ export interface ClassifySeverityOptions extends Pick<
| "signal"
| "workingDirectory"
> {
- /** Classification policy. Omit to inherit existing severity without a model call. */
+ /** Classification policy override. Omit to use the built-in severity rubric. */
rubricPath?: string;
+ /** Independent reviews per finding: integer 1–10, default 3. */
+ classificationRuns?: number;
/** Supporting evidence, separate from classification policy. */
knowledgeBasePaths?: readonly string[];
/** @internal Test client for the shared read-only runtime. */
@@ -50,6 +54,8 @@ export interface SeverityAssessment extends z.infer<typeof decisionSchema> {
export interface SeverityClassification {
schemaVersion: 1;
assessedAt: string;
+ /** Absent on legacy results whose review count was not recorded. */
+ classificationRuns?: number;
rubricSha256: string | null;
knowledgeBaseSha256: string | null;
assessments: SeverityAssessment[];
@@ -65,7 +71,6 @@ export interface SeverityClassificationCheckpoint {
): Promise<void>;
}
-const levelSchema = z.enum(SCAN_SEVERITIES);
const textSchema = z
.string()
.min(1)
@@ -75,6 +80,7 @@ const digestSchema = z.string().regex(/^[a-f0-9]{64}$/u);
export const severityClassificationSchema = z.strictObject({
schemaVersion: z.literal(1),
assessedAt: z.string().datetime(),
+ classificationRuns: classificationRunsSchema.optional(),
rubricSha256: digestSchema.nullable(),
knowledgeBaseSha256: digestSchema.nullable(),
assessments: z.array(
@@ -102,6 +108,9 @@ export async function classifySeverityInternal(
checkpoint?: SeverityClassificationCheckpoint,
): Promise<SeverityClassification> {
options.signal?.throwIfAborted();
+ const classificationRuns = classificationRunsSchema
+ .default(3)
+ .parse(options.classificationRuns);
const ids = new Set<string>();
for (const finding of findings) {
if (!finding.findingId?.trim() || ids.has(finding.findingId)) {
@@ -113,15 +122,16 @@ export async function classifySeverityInternal(
}
const rubric =
options.rubricPath === undefined
- ? null
+ ? [defaultSeverityRubric]
: await readDocuments([options.rubricPath], options.signal);
const knowledge = options.knowledgeBasePaths?.length
? await readDocuments(options.knowledgeBasePaths, options.signal)
: null;
const result: SeverityClassification = {
schemaVersion: 1,
+ classificationRuns,
assessedAt: new Date().toISOString(),
- rubricSha256: rubric === null ? null : workflowDigest(rubric),
+ rubricSha256: workflowDigest(rubric),
knowledgeBaseSha256: knowledge === null ? null : workflowDigest(knowledge),
assessments: [],
};
@@ -137,31 +147,14 @@ export async function classifySeverityInternal(
result.assessments.push(previous);
continue;
}
- let decision: z.infer<typeof decisionSchema>;
- if (rubric === null) {
- const parsed = levelSchema.safeParse(finding.severity?.level);
- if (!parsed.success) {
- throw new CodexSecurityError(
- `Finding ${finding.findingId} has no existing severity; supply a rubric.`,
- );
- }
- decision = {
- findingId: finding.findingId,
- decision: "assessed",
- level: parsed.data,
- rubricLabel: null,
- rationale:
- finding.severity?.rationale?.trim() ||
- "Inherited the finding's existing severity.",
- confidence: null,
- reviewTrigger: finding.severity?.changeConditions?.trim() || null,
- };
- } else {
+ const prompt = [
+ severityReviewInstructions,
+ JSON.stringify({ rubric, knowledgeBase: knowledge, finding }),
+ ].join("\n\n");
+ const decision = await reviewSeverity(async () => {
+ options.signal?.throwIfAborted();
const response = await runReadOnlyCodex(
- [
- severityReviewInstructions,
- JSON.stringify({ rubric, knowledgeBase: knowledge, finding }),
- ].join("\n\n"),
+ prompt,
z.toJSONSchema(decisionSchema),
options,
{
@@ -172,22 +165,20 @@ export async function classifySeverityInternal(
);
options.signal?.throwIfAborted();
try {
- decision = parseSeverityDecision(
- JSON.parse(response),
- finding.findingId,
- );
+ return parseSeverityDecision(JSON.parse(response), finding.findingId);
} catch (error) {
throw new CodexSecurityError(
"Severity classification returned an invalid assessment.",
{ cause: error },
);
}
- }
+ }, classificationRuns);
+ options.signal?.throwIfAborted();
const assessment: SeverityAssessment = {
...decision,
occurrenceId: finding.occurrenceId ?? null,
inputSha256,
- source: rubric === null ? "existing-severity" : "rubric",
+ source: "rubric",
};
await checkpoint?.save(finding, assessment, result);
result.assessments.push(assessment);
diff --git a/sdk/typescript/src/cli.ts b/sdk/typescript/src/cli.ts
index 965544acb..27e52285f 100644
--- a/sdk/typescript/src/cli.ts
+++ b/sdk/typescript/src/cli.ts
@@ -1,4 +1,5 @@
#!/usr/bin/env node
+import { classificationRunsSchema } from "./severity-review.js";
import { workbenchArgs } from "./workbench-arguments.js";
import { listenForAbort } from "./cli-signals.js";
import { appServerCyberAccessProgram } from "./cli-daybreak.js";
@@ -50,6 +51,7 @@ import { Readable, Writable as NodeWritable } from "node:stream";
import { fileURLToPath, pathToFileURL } from "node:url";
import {
isDeepStrictEqual,
+ parseArgs,
promisify,
stripVTControlCharacters,
} from "node:util";
@@ -1877,7 +1879,29 @@ export async function main(
!argv.includes("-h") &&
!argv.includes("--schema")
) {
- if (argv.length !== 2 || (argv[0] === "scan" && argv[1] !== "--host")) {
+ let classificationRuns: number | undefined;
+ if (argv[0] === "classify-severity") {
+ try {
+ const { values } = parseArgs({
+ args: argv.slice(1),
+ options: {
+ records: { type: "boolean" },
+ runs: { type: "string" },
+ },
+ });
+ classificationRuns = classificationRunsSchema.parse(
+ values.runs === undefined ? 3 : Number(values.runs),
+ );
+ } catch {
+ errorOutput.write(
+ "codex-security: Use --records with only --runs (an integer from 1 to 10; default: 3).\n",
+ );
+ return 2;
+ }
+ } else if (
+ argv.length !== 2 ||
+ (argv[0] === "scan" && argv[1] !== "--host")
+ ) {
errorOutput.write(
argv[0] === "scan"
? "codex-security: --host must be used alone with scan.\n"
@@ -1901,10 +1925,10 @@ export async function main(
dependencies.recordsInput ?? process.stdin,
output,
(value, options) =>
- classifySeverityRecords(
- value as ClassifySeverityRecordsInput,
- options,
- ),
+ classifySeverityRecords(value as ClassifySeverityRecordsInput, {
+ ...options,
+ classificationRuns,
+ }),
controller.signal,
)
: await runProtocol(
@@ -4116,15 +4140,20 @@ export async function main(
})
.command("classify-severity", {
description: "Assess severity for saved findings or canonical records.",
- hint: "Use --rubric for saved findings or --records alone for host-provided classification.",
+ hint: "Use --rubric to override the built-in policy or --records for host-provided classification.",
destructive: true,
mcp: false,
options: z.object({
+ runs: classificationRunsSchema
+ .default(3)
+ .describe(
+ "Number of independent classification runs (1–10; default: 3).",
+ ),
records: z
.boolean()
.default(false)
.describe(
- "Classify canonical records through host-provided reviews over version 1 JSON-RPC; use alone.",
+ "Classify canonical records through host-provided reviews over version 1 JSON-RPC; accepts --runs.",
),
reprocess: z
.boolean()
@@ -4141,7 +4170,7 @@ export async function main(
rubric: optionValue("--rubric")
.optional()
.describe(
- "Classification policy document; omit to inherit existing severity without a model call.",
+ "Classification policy document; omit to classify with the built-in severity rubric.",
),
knowledgeBase: z
.array(optionValue("--knowledge-base"))
@@ -4192,6 +4221,7 @@ export async function main(
findingIds:
options.findingId.length === 0 ? undefined : options.findingId,
reprocess: options.reprocess,
+ classificationRuns: options.runs,
model: options.model,
reasoningEffort: options.effort,
};
diff --git a/sdk/typescript/src/default-severity-rubric.ts b/sdk/typescript/src/default-severity-rubric.ts
new file mode 100644
index 000000000..6c1c5bb20
--- /dev/null
+++ b/sdk/typescript/src/default-severity-rubric.ts
@@ -0,0 +1,66 @@
+/** Built into the CLI and SDK so classification needs no external policy file. */
+export const defaultSeverityRubric = `# Severity guidance
+
+Judge the attacker's actual starting access, the security boundary crossed, and
+the additional unauthorized harm. Use the supplied report and evidence; do not
+perform new validation. Keep confidence separate from severity and identify the
+missing fact that could change the assessment. Missing proof alone does not
+establish that a broader attack is impossible; do not invent an attack path or
+downgrade merely because exploitation was not demonstrated.
+
+## Critical
+
+Reserve Critical for consequential, strongly supported compromise meeting all
+three conditions:
+- Access: an unauthenticated internet attacker, ordinary external user, or
+ publicly admitted untrusted workload can exploit the interface without
+ employee-only access, an invitation, an approved business relationship, or
+ compromise of a trusted supplier.
+- Impact: crossing an important security boundary enables consequential access
+ to protected data, privileged actions, meaningful code execution, workload
+ escape, or comparable compromise of the affected system.
+- Evidence: demonstrated exploitation or strong source-backed validation
+ establishes the complete attack path.
+
+Second-order attacks can qualify: an external attacker plants data that a trusted
+user or system later processes, causing consequential unauthorized access or
+privileged actions.
+
+## High
+
+Assign High when a realistic, evidenced path gives a lower-trust attacker a
+substantial unauthorized capability they did not already possess. Internal
+users, partners, invited collaborators, and admitted untrusted workloads can
+qualify; unauthenticated internet access is not required. Consequences include
+sensitive customer or business data or reusable credentials, account takeover,
+meaningful cross-tenant access or actions as another user, control of production
+workloads or deployments, privileged identities, encryption keys or important
+isolation boundaries, or significant repeatable disruption of a critical or
+shared service. A dangerous function, production label, internal access, or the
+existence of a privileged credential alone is insufficient: identify the
+lower-trust actor, boundary crossing, and consequential new capability.
+
+## Medium
+
+Assign Medium to a credible vulnerability with concretely bounded impact, such
+as nonsecret operational information, narrow unauthorized changes, contained
+disruption of a noncritical service, or isolated development or staging systems
+without a consequential pivot. A fully demonstrated vulnerability can still be
+Medium. Base a downgrade on verified constraints, not on an assumption that
+missing evidence rules out broader harm.
+
+## Low
+
+Assign Low to a supported security weakness with minor additional unauthorized
+harm and verified limits on its scope, sensitivity, or duration, such as a small
+information exposure with little security value or a narrowly contained nuisance.
+Do not use Low as a substitute for uncertainty about a potentially serious path.
+
+## Informational
+
+Assign Informational to a hardening opportunity or observation for which the
+supplied evidence establishes no additional unauthorized capability or security
+impact. Explain why it is informational rather than treating missing proof of
+exploitation as proof of safety. Administrative records that do not describe a
+security finding may be excluded; exclusion is distinct from severity.
+`;
diff --git a/sdk/typescript/src/severity-review.ts b/sdk/typescript/src/severity-review.ts
index 71d89542e..3d8aa1d9c 100644
--- a/sdk/typescript/src/severity-review.ts
+++ b/sdk/typescript/src/severity-review.ts
@@ -2,6 +2,8 @@ import { SCAN_SEVERITIES } from "./scan-settings.js";
import { z } from "zod";
import { CodexSecurityError } from "./errors.js";
+export const classificationRunsSchema = z.number().int().min(1).max(10);
+
const levelSchema = z.enum(SCAN_SEVERITIES);
const textSchema = z
.string()
@@ -58,3 +60,25 @@ export function parseSeverityDecision(
);
}
}
+
+/** Run independent reviews and retain the first complete answer for the winning level. */
+export async function reviewSeverity(
+ review: () => Promise<SeverityDecision>,
+ classificationRuns = 3,
+): Promise<SeverityDecision> {
+ const runs = classificationRunsSchema.parse(classificationRuns);
+ const decisions: SeverityDecision[] = [];
+ for (let run = 0; run < runs; run++) decisions.push(await review());
+ // Exclusion is a distinct vote; any assessed severity wins a tied vote.
+ const levels = [...SCAN_SEVERITIES, null];
+ const count = (level: SeverityDecision["level"]) =>
+ decisions.filter((decision) => decision.level === level).length;
+ return decisions.reduce((winner, decision) => {
+ const difference = count(decision.level) - count(winner.level);
+ return difference > 0 ||
+ (difference === 0 &&
+ levels.indexOf(decision.level) < levels.indexOf(winner.level))
+ ? decision
+ : winner;
+ });
+}
diff --git a/sdk/typescript/src/severity-store.ts b/sdk/typescript/src/severity-store.ts
index 040d132fd..ab3a2cbfd 100644
--- a/sdk/typescript/src/severity-store.ts
+++ b/sdk/typescript/src/severity-store.ts
@@ -41,6 +41,7 @@ export class SeverityStore {
scanId,
findingIds,
assessedAt: result.assessedAt,
+ classificationRuns: result.classificationRuns,
rubricSha256: result.rubricSha256,
knowledgeBaseSha256: result.knowledgeBaseSha256,
});
@@ -58,6 +59,7 @@ export class SeverityStore {
finding,
assessment: {
...assessment,
+ classificationRuns: result.classificationRuns,
rubricSha256: result.rubricSha256,
knowledgeBaseSha256: result.knowledgeBaseSha256,
},
@@ -85,6 +87,8 @@ export class SeverityStore {
{
schemaVersion: 1,
assessedAt: saved["assessedAt"] as string,
+ classificationRuns: (saved["classificationRuns"] ?? undefined) as
+ number | undefined,
rubricSha256: saved["rubricSha256"] as string | null,
knowledgeBaseSha256: saved["knowledgeBaseSha256"] as string | null,
},
@@ -139,12 +143,15 @@ function matchingClassification(
assessments: stored
.filter(
(row) =>
+ (row["classificationRuns"] ?? undefined) ===
+ result.classificationRuns &&
row["rubricSha256"] === result.rubricSha256 &&
row["knowledgeBaseSha256"] === result.knowledgeBaseSha256,
)
.map(
({
assessedAt: _assessedAt,
+ classificationRuns: _runs,
rubricSha256: _rubric,
knowledgeBaseSha256: _knowledge,
...assessment
diff --git a/sdk/typescript/tests-ts/classify-scan-severity.test.ts b/sdk/typescript/tests-ts/classify-scan-severity.test.ts
index 5d1587d28..40a85efa2 100644
--- a/sdk/typescript/tests-ts/classify-scan-severity.test.ts
+++ b/sdk/typescript/tests-ts/classify-scan-severity.test.ts
@@ -180,6 +180,7 @@ test("classification and saved assessments run with Node and no Python", async (
const { scanId: _scanId, ...classification } =
await classifyScanDirectorySeverity(scanDirectory, {
environment: withoutPython,
+ codex: recordingClassifier().codex,
});
expect(
await readScanSeverityClassification(
@@ -196,7 +197,10 @@ test("saved assessments retain embedded NULs in imported scan IDs", async () =>
const { environment, scanDirectory, scanId, findings } =
await fixture("imported\0scan");
const { scanId: classifiedScanId, ...classification } =
- await classifyScanDirectorySeverity(scanDirectory, { environment });
+ await classifyScanDirectorySeverity(scanDirectory, {
+ environment,
+ codex: recordingClassifier().codex,
+ });
expect(classifiedScanId).toBe(scanId);
expect(
await readScanSeverityClassification(
@@ -235,7 +239,7 @@ test("severity helpers skip Node executables in the caller's enclosing checkout"
[
"--eval",
`import { classifyScanDirectorySeverity } from ${JSON.stringify(entryPoint.href)};
-await classifyScanDirectorySeverity(${JSON.stringify(scanDirectory)}, { environment: process.env });`,
+await classifyScanDirectorySeverity(${JSON.stringify(scanDirectory)}, { environment: process.env, findingIds: [] });`,
],
{
cwd,
@@ -374,13 +378,17 @@ test("saved severity protects repository and caller tools across concurrent look
function recordingClassifier() {
const calls: string[] = [];
- const control = { failOn: "", excluded: false };
+ const control = { failOn: "", failOnRun: 1, excluded: false };
const codex = codexWithRun(async (prompt: string) => {
const { finding } = JSON.parse(prompt.split("\n\n").at(-1)!) as {
finding: Finding;
};
calls.push(finding.findingId);
- if (finding.findingId === control.failOn)
+ if (
+ finding.findingId === control.failOn &&
+ calls.filter((id) => id === finding.findingId).length ===
+ control.failOnRun
+ )
throw new Error("Interrupted model call");
return classifier(finding, control.excluded).startThread().run();
});
@@ -388,10 +396,11 @@ function recordingClassifier() {
}
test("checkpoints each finding, resumes missing work, and reprocesses only the selection", async () => {
- const { environment, scanDirectory, rubricPath, findings } = await fixture();
+ const { environment, scanDirectory, findings } = await fixture();
const { codex, calls, control } = recordingClassifier();
- const options = { environment, rubricPath, codex };
+ const options = { environment, codex };
control.failOn = findings[1]!.findingId;
+ control.failOnRun = 3;
await expect(
classifyScanDirectorySeverity(scanDirectory, options),
).rejects.toThrow("Interrupted model call");
@@ -409,7 +418,7 @@ test("checkpoints each finding, resumes missing work, and reprocesses only the s
control.failOn = "";
calls.length = 0;
const complete = await classifyScanDirectorySeverity(scanDirectory, options);
- expect(calls).toEqual([findings[1]!.findingId]);
+ expect(calls).toEqual(Array(3).fill(findings[1]!.findingId));
expect(complete.assessments).toHaveLength(2);
const { scanId: _scanId, ...assessment } = complete;
expect(
@@ -444,7 +453,7 @@ test("checkpoints each finding, resumes missing work, and reprocesses only the s
reprocess: true,
};
const revised = await classifyScanDirectorySeverity(scanDirectory, selected);
- expect(calls).toEqual([findings[0]!.findingId]);
+ expect(calls).toEqual(Array(3).fill(findings[0]!.findingId));
expect(revised.assessments[0]!.decision).toBe("excluded");
const revisedRows = await query(
environment,
@@ -470,6 +479,7 @@ test("checkpoints each finding, resumes missing work, and reprocesses only the s
).toHaveLength(1);
control.failOn = findings[0]!.findingId;
+ control.failOnRun = 2;
await expect(
classifyScanDirectorySeverity(scanDirectory, selected),
).rejects.toThrow("Interrupted model call");
@@ -555,10 +565,22 @@ test("migration leaves unindexed legacy assessments incomplete until reclassifie
const environment = first.environment;
const { scanId, ...classification } = await classifyScanDirectorySeverity(
first.scanDirectory,
- { environment },
+ { environment, codex: recordingClassifier().codex },
);
await query(environment, "DROP TABLE scan_severity_assessments");
- await query(environment, "DELETE FROM schema_migrations WHERE version = 43");
+ for (const table of [
+ "finding_severity_assessments",
+ "scan_severity_classifications",
+ ]) {
+ await query(
+ environment,
+ `ALTER TABLE ${table} DROP COLUMN classification_runs`,
+ );
+ }
+ await query(
+ environment,
+ "DELETE FROM schema_migrations WHERE version IN (43, 52)",
+ );
expect(
await readScanSeverityClassification(
first.scanDirectory,
@@ -567,14 +589,17 @@ test("migration leaves unindexed legacy assessments incomplete until reclassifie
undefined,
environment,
),
- ).toEqual(classification);
+ ).toEqual({ ...classification, classificationRuns: undefined });
expect(
await query(
environment,
"SELECT version FROM schema_migrations WHERE version = 43",
),
).toEqual([]);
- await classifyScanDirectorySeverity(second.scanDirectory, { environment });
+ await classifyScanDirectorySeverity(second.scanDirectory, {
+ environment,
+ codex: recordingClassifier().codex,
+ });
await expect(
readScanSeverityClassification(
first.scanDirectory,
@@ -585,8 +610,12 @@ test("migration leaves unindexed legacy assessments incomplete until reclassifie
),
).rejects.toThrow("incomplete");
expect(
- (await classifyScanDirectorySeverity(first.scanDirectory, { environment }))
- .assessments,
+ (
+ await classifyScanDirectorySeverity(first.scanDirectory, {
+ environment,
+ codex: recordingClassifier().codex,
+ })
+ ).assessments,
).toEqual(classification.assessments);
expect(
await query(
@@ -609,17 +638,17 @@ test("changed rubric, context, or evidence invalidates matching checkpoints", as
calls.length = 0;
await writeFile(rubricPath, "Assign Medium to bounded metadata reads.");
await classifyScanDirectorySeverity(scanDirectory, options);
- expect(calls).toHaveLength(2);
+ expect(calls).toHaveLength(6);
const context = join(root, "context.md");
await writeFile(context, "The system contains operational counters.");
calls.length = 0;
const withContext = { ...options, knowledgeBasePaths: [context] };
await classifyScanDirectorySeverity(scanDirectory, withContext);
- expect(calls).toHaveLength(2);
+ expect(calls).toHaveLength(6);
calls.length = 0;
await writeFile(context, "The counters include protected metadata.");
await classifyScanDirectorySeverity(scanDirectory, withContext);
- expect(calls).toHaveLength(2);
+ expect(calls).toHaveLength(6);
const findingPath = join(scanDirectory, "findings.json");
const document = JSON.parse(
@@ -641,7 +670,7 @@ test("changed rubric, context, or evidence invalidates matching checkpoints", as
).rejects.toThrow("does not match");
calls.length = 0;
await classifyScanDirectorySeverity(scanDirectory, withContext);
- expect(calls).toEqual([findings[0]!.findingId]);
+ expect(calls).toEqual(Array(3).fill(findings[0]!.findingId));
expect(
await query(environment, "SELECT * FROM findings ORDER BY id"),
).toEqual(originalFindings);
@@ -772,7 +801,10 @@ test("exclusions and empty dedupe selections do not create tickets", async () =>
test("failed or canceled reassessment leaves the last successful assessment intact", async () => {
const { environment, scanDirectory, rubricPath, findings } = await fixture();
- await classifyScanDirectorySeverity(scanDirectory, { environment });
+ await classifyScanDirectorySeverity(scanDirectory, {
+ environment,
+ codex: recordingClassifier().codex,
+ });
const path = join(scanDirectory, "severity-classification.json");
const before = await readFile(path);
await expect(
@@ -805,6 +837,7 @@ test("publication reads SQLite even when the JSON export is modified or symlinke
const { environment, root, scanDirectory, findings } = await fixture();
const result = await classifyScanDirectorySeverity(scanDirectory, {
environment,
+ codex: recordingClassifier().codex,
});
const path = join(scanDirectory, "severity-classification.json");
await writeFile(path, JSON.stringify({ ...result, scanId: "other-scan" }));
@@ -816,9 +849,10 @@ test("publication reads SQLite even when the JSON export is modified or symlinke
})
).issues,
).toHaveLength(findings.length);
- const stale = await classifySeverity([
- { ...findings[0]!, summary: "Different report" },
- ]);
+ const stale = await classifySeverity(
+ [{ ...findings[0]!, summary: "Different report" }],
+ { codex: classifier(findings[0]!) },
+ );
await expect(
prepareScanPublication(scanDirectory, {
environment,
@@ -838,7 +872,10 @@ test("publication reads SQLite even when the JSON export is modified or symlinke
})
).issues,
).toHaveLength(findings.length);
- await classifyScanDirectorySeverity(scanDirectory, { environment });
+ await classifyScanDirectorySeverity(scanDirectory, {
+ environment,
+ codex: recordingClassifier().codex,
+ });
expect(await readFile(external, "utf8")).toBe(JSON.stringify(result));
});
@@ -916,7 +953,7 @@ test.each(["latest", "scan_prefix"])(
const seen: string[][] = [];
const result = await classifyScanSeverityInternal(
selector,
- { environment },
+ { environment, codex: recordingClassifier().codex },
{
currentDirectory: () => scanDirectory,
runWorkbench: async (args): Promise<JsonObject> => {
@@ -950,7 +987,10 @@ test.each(["latest", "scan_prefix"])(
test("migrates existing databases without changing findings and reads older state without writes", async () => {
const { environment, scanDirectory } = await fixture();
- await classifyScanDirectorySeverity(scanDirectory, { environment });
+ await classifyScanDirectorySeverity(scanDirectory, {
+ environment,
+ codex: recordingClassifier().codex,
+ });
const original = await query(
environment,
"SELECT * FROM findings ORDER BY id",
@@ -960,7 +1000,7 @@ test("migrates existing databases without changing findings and reads older stat
await query(environment, "DROP TABLE scan_severity_classifications");
await query(
environment,
- "DELETE FROM schema_migrations WHERE version IN (41, 43)",
+ "DELETE FROM schema_migrations WHERE version IN (41, 43, 52)",
);
expect(
(
@@ -976,7 +1016,10 @@ test("migrates existing databases without changing findings and reads older stat
"SELECT version FROM schema_migrations WHERE version = 41",
),
).toEqual([]);
- await classifyScanDirectorySeverity(scanDirectory, { environment });
+ await classifyScanDirectorySeverity(scanDirectory, {
+ environment,
+ codex: recordingClassifier().codex,
+ });
expect(
await query(
environment,
@@ -1020,7 +1063,7 @@ test("classifying another scan preserves both recurring-finding assessments", as
};
await classifyScanDirectorySeverity(first.scanDirectory, options);
await classifyScanDirectorySeverity(second.scanDirectory, options);
- expect(calls).toHaveLength(4);
+ expect(calls).toHaveLength(12);
calls.length = 0;
for (const scan of [first, second, first, second]) {
expect(
@@ -1035,3 +1078,143 @@ test("classifying another scan preserves both recurring-finding assessments", as
}
expect(calls).toEqual([]);
});
+
+test("default rubric replaces legacy inherited assessments and respects custom policy changes", async () => {
+ const { environment, scanDirectory, rubricPath, findings, scanId } =
+ await fixture();
+ const { codex, calls } = recordingClassifier();
+ const options = { environment, codex };
+ await classifyScanDirectorySeverity(scanDirectory, options);
+ // Recreate the provenance emitted by the former no-rubric inheritance path.
+ for (const table of [
+ "scan_severity_assessments",
+ "finding_severity_assessments",
+ ]) {
+ await query(
+ environment,
+ `UPDATE ${table} SET rubric_sha256 = NULL, source = 'existing-severity', level = 'high', rubric_label = NULL, confidence = NULL`,
+ );
+ }
+ await query(
+ environment,
+ "UPDATE scan_severity_classifications SET rubric_sha256 = NULL",
+ );
+ const legacy = await readScanSeverityClassification(
+ scanDirectory,
+ scanId,
+ findings,
+ undefined,
+ environment,
+ );
+ expect(
+ legacy?.assessments.every((a) => a.source === "existing-severity"),
+ ).toBe(true);
+ calls.length = 0;
+ const result = await classifyScanDirectorySeverity(scanDirectory, options);
+ expect(calls).toHaveLength(6);
+ expect(result.rubricSha256).toMatch(/^[a-f0-9]{64}$/u);
+ expect(
+ result.assessments.every(
+ (a) => a.source === "rubric" && a.level === "medium",
+ ),
+ ).toBe(true);
+ calls.length = 0;
+ expect(
+ (await classifyScanDirectorySeverity(scanDirectory, options)).assessments,
+ ).toEqual(result.assessments);
+ expect(calls).toHaveLength(0);
+ const custom = await classifyScanDirectorySeverity(scanDirectory, {
+ ...options,
+ rubricPath,
+ });
+ expect(calls).toHaveLength(6);
+ expect(custom.rubricSha256).not.toBe(result.rubricSha256);
+ calls.length = 0;
+ expect(
+ (await classifyScanDirectorySeverity(scanDirectory, options)).rubricSha256,
+ ).toBe(result.rubricSha256);
+ expect(calls).toHaveLength(6);
+});
+
+test("saved assessments bind their run count and invalidate changed or unrecorded counts", async () => {
+ const { environment, scanDirectory, scanId, findings } = await fixture();
+ const { codex, calls, control } = recordingClassifier();
+ for (const [classificationRuns, expectedCalls] of [
+ [3, 6],
+ [3, 0],
+ [1, 2],
+ [1, 0],
+ [10, 20],
+ [3, 6],
+ ]) {
+ calls.length = 0;
+ const result = await classifyScanDirectorySeverity(scanDirectory, {
+ environment,
+ codex,
+ classificationRuns,
+ });
+ expect(calls).toHaveLength(expectedCalls!);
+ expect(result.classificationRuns).toBe(classificationRuns);
+ const saved = await readScanSeverityClassification(
+ scanDirectory,
+ scanId,
+ findings,
+ undefined,
+ environment,
+ );
+ expect(saved?.classificationRuns).toBe(classificationRuns);
+ }
+ for (const table of [
+ "scan_severity_classifications",
+ "scan_severity_assessments",
+ "finding_severity_assessments",
+ ]) {
+ await query(environment, `UPDATE ${table} SET classification_runs = NULL`);
+ }
+ expect(
+ (
+ await readScanSeverityClassification(
+ scanDirectory,
+ scanId,
+ findings,
+ undefined,
+ environment,
+ )
+ )?.classificationRuns,
+ ).toBeUndefined();
+ calls.length = 0;
+ await classifyScanDirectorySeverity(scanDirectory, { environment, codex });
+ expect(calls).toHaveLength(6);
+ expect(
+ await query(
+ environment,
+ `SELECT classification_runs FROM scan_severity_classifications WHERE scan_id = '${scanId}'`,
+ ),
+ ).toEqual([{ classification_runs: 3 }]);
+ calls.length = 0;
+ control.failOn = findings[1]!.findingId;
+ await expect(
+ classifyScanDirectorySeverity(scanDirectory, {
+ environment,
+ codex,
+ classificationRuns: 1,
+ }),
+ ).rejects.toThrow("Interrupted model call");
+ await expect(
+ readScanSeverityClassification(
+ scanDirectory,
+ scanId,
+ findings,
+ undefined,
+ environment,
+ ),
+ ).rejects.toThrow("incomplete");
+ calls.length = 0;
+ control.failOn = "";
+ await classifyScanDirectorySeverity(scanDirectory, {
+ environment,
+ codex,
+ classificationRuns: 1,
+ });
+ expect(calls).toEqual([findings[1]!.findingId]);
+});
diff --git a/sdk/typescript/tests-ts/classify-severity-records.test.ts b/sdk/typescript/tests-ts/classify-severity-records.test.ts
index 56a8fac07..46bf803ee 100644
--- a/sdk/typescript/tests-ts/classify-severity-records.test.ts
+++ b/sdk/typescript/tests-ts/classify-severity-records.test.ts
@@ -1,3 +1,4 @@
+import { defaultSeverityRubric } from "../src/default-severity-rubric.js";
import { expect, test } from "bun:test";
import { createHash } from "node:crypto";
import { PassThrough, Writable } from "node:stream";
@@ -11,7 +12,9 @@ import { capture, dependencies } from "./cli-fixtures.js";
const hash = (value: string) =>
createHash("sha256").update(value, "utf8").digest("hex");
-function input(): ClassifySeverityRecordsInput {
+function input(): ClassifySeverityRecordsInput & {
+ rubric: NonNullable<ClassifySeverityRecordsInput["rubric"]>;
+} {
const markdown =
"# Severity\nHigh: unauthorized access to private records.\n";
const findingJson =
@@ -53,6 +56,9 @@ test("binds assessments to exact UTF-8 snapshots without rewriting the original
requests.push(request);
expect(request.model).toBe(original.model);
expect(request.effort).toBe("high");
+ expect(JSON.parse(request.prompt).rubric).toEqual([
+ original.rubric.markdown,
+ ]);
expect(request.trustedInstructions).toContain("Do not use tools");
expect(request.schema).toHaveProperty("required");
expect(request.prompt).toContain(original.records[0]!.findingJson);
@@ -76,7 +82,9 @@ test("binds assessments to exact UTF-8 snapshots without rewriting the original
],
});
expect(original).toEqual(before);
- expect(requests).toHaveLength(1);
+ expect(requests).toHaveLength(3);
+ expect(new Set(requests.map(({ requestId }) => requestId)).size).toBe(3);
+ expect(new Set(requests.map(({ prompt }) => prompt)).size).toBe(1);
expect(result.assessments[0]).not.toHaveProperty("findingId");
});
@@ -143,7 +151,7 @@ test.each([
async run(request) {
calls++;
const result: Record<string, unknown> = answer(request);
- if (calls === 1) return result;
+ if (calls <= 3) return result;
if (scenario === "host-failure")
throw new Error("Host could not complete review");
if (scenario === "wrong-id") result["findingId"] = "canonical-one";
@@ -164,7 +172,7 @@ test.each([
scenario === "host-failure" ? "review_failed" : "invalid_assessment",
},
});
- expect(calls).toBe(2);
+ expect(calls).toBe(4);
});
test("exclusion is distinct from low severity", async () => {
@@ -224,6 +232,9 @@ test("cancellation rejects even when the host review does not settle", async ()
test.each([
"completed",
+ "default-rubric",
+ "one-run",
+ "ten-runs",
"host-error",
"wrong-response",
"disconnect",
@@ -272,23 +283,47 @@ test.each([
throw new Error("Local scan execution is forbidden");
};
const result = main(
- ["classify-severity", "--records"],
+ [
+ "classify-severity",
+ "--records",
+ ...(scenario === "one-run"
+ ? ["--runs", "1"]
+ : scenario === "ten-runs"
+ ? ["--runs=10"]
+ : []),
+ ],
output,
capture().stream,
deps,
);
+ const params: ClassifySeverityRecordsInput = input();
+ if (scenario === "default-rubric") delete params.rubric;
+ const completed = [
+ "completed",
+ "default-rubric",
+ "one-run",
+ "ten-runs",
+ ].includes(scenario);
stream.write(
- `${JSON.stringify({ jsonrpc: "2.0", id: "run-one", method: "run", params: input() })}\n`,
+ `${JSON.stringify({ jsonrpc: "2.0", id: "run-one", method: "run", params })}\n`,
);
expect(await result).toBe(
- scenario === "completed" ? 0 : scenario === "host-error" ? 1 : 2,
+ completed ? 0 : scenario === "host-error" ? 1 : 2,
);
expect(
messages.filter((message) => message.method === "review.run"),
- ).toHaveLength(1);
+ ).toHaveLength(
+ completed
+ ? scenario === "one-run"
+ ? 1
+ : scenario === "ten-runs"
+ ? 10
+ : 3
+ : 1,
+ );
const final = messages.at(-1);
expect(final.id).toBe("run-one");
- if (scenario === "completed")
+ if (completed)
expect(final.result.assessments[0]).toHaveProperty("id", "canonical-one");
else if (scenario === "host-error")
expect(final.result).toMatchObject({
@@ -336,6 +371,7 @@ test("empty records probe version 1 without host execution", async () => {
}),
).toEqual({
version: 1,
+ classificationRuns: 3,
status: "completed",
rubric: { revision: data.rubric.revision, sha256: data.rubric.sha256 },
assessments: [],
@@ -359,3 +395,135 @@ test("immediate SDK cancellation never starts a queued host review", async () =>
await expect(result).rejects.toThrow("Stopped before review");
expect(calls).toBe(0);
});
+
+test("canonical records use majority voting and highest severity for a three-way tie", async () => {
+ for (const [levels, expected] of [
+ [["low", "high", "low"], "low"],
+ [["medium", "critical", "high"], "critical"],
+ ] as const) {
+ let calls = 0;
+ const result = await classifySeverityRecords(input(), {
+ reviewRunner: {
+ async run(request) {
+ return { ...answer(request), level: levels[calls++] };
+ },
+ },
+ });
+ expect(calls).toBe(3);
+ expect(result.assessments[0]?.level).toBe(expected);
+ }
+});
+
+test.each([2, 3, 10])(
+ "canonical review %s must succeed before returning any assessment",
+ async (failedRun) => {
+ for (const invalid of [false, true]) {
+ let calls = 0;
+ const data: ClassifySeverityRecordsInput = input();
+ delete data.rubric;
+ const result = await classifySeverityRecords(data, {
+ classificationRuns: Math.max(3, failedRun),
+ reviewRunner: {
+ async run(request) {
+ if (++calls !== failedRun) return answer(request);
+ if (invalid)
+ return { ...answer(request), findingId: "wrong-finding" };
+ throw new Error("Host review failed");
+ },
+ },
+ });
+ expect(result).toMatchObject({
+ status: "unresolved",
+ assessments: [],
+ error: {
+ recordId: "canonical-one",
+ code: invalid ? "invalid_assessment" : "review_failed",
+ },
+ });
+ expect(calls).toBe(failedRun);
+ }
+ },
+);
+
+test.each([2, 3])(
+ "canonical cancellation during review %s never returns a partial vote",
+ async (canceledRun) => {
+ const controller = new AbortController();
+ let calls = 0;
+ await expect(
+ classifySeverityRecords(input(), {
+ signal: controller.signal,
+ reviewRunner: {
+ async run(request) {
+ if (++calls === canceledRun) controller.abort(new Error("Stopped"));
+ return answer(request);
+ },
+ },
+ }),
+ ).rejects.toThrow("Stopped");
+ expect(calls).toBe(canceledRun);
+ },
+);
+
+test("omitted canonical rubric uses the bundled policy and returns its exact binding", async () => {
+ const data: ClassifySeverityRecordsInput = input();
+ delete data.rubric;
+ const requests: SeverityClassificationReviewRequest[] = [];
+ const result = await classifySeverityRecords(data, {
+ reviewRunner: {
+ async run(request) {
+ requests.push(request);
+ expect(JSON.parse(request.prompt).rubric).toEqual([
+ defaultSeverityRubric,
+ ]);
+ return answer(request);
+ },
+ },
+ });
+ expect(requests).toHaveLength(3);
+ expect(new Set(requests.map(({ requestId }) => requestId)).size).toBe(3);
+ expect(result).toMatchObject({
+ status: "completed",
+ rubric: { revision: 1, sha256: hash(defaultSeverityRubric) },
+ assessments: [{ level: "high" }],
+ });
+ expect(data.rubric).toBeUndefined();
+});
+
+test.each([1, 2, 4, 10])(
+ "runs %i independent canonical reviews",
+ async (classificationRuns) => {
+ let calls = 0;
+ const result = await classifySeverityRecords(input(), {
+ classificationRuns,
+ reviewRunner: {
+ async run(request) {
+ calls++;
+ return answer(request);
+ },
+ },
+ });
+ expect(calls).toBe(classificationRuns);
+ expect(result.classificationRuns).toBe(classificationRuns);
+ expect(result.status).toBe("completed");
+ },
+);
+
+test.each([0, -1, 11, 1.5, NaN, Infinity])(
+ "rejects invalid canonical run count %s before host work",
+ async (classificationRuns) => {
+ let calls = 0;
+ await expect(
+ classifySeverityRecords(input(), {
+ classificationRuns,
+ reviewRunner: {
+ async run(request) {
+ calls++;
+ return answer(request);
+ },
+ },
+ }),
+ ).rejects.toThrow();
+ expect(calls).toBe(0);
+ },
+);
diff --git a/sdk/typescript/tests-ts/classify-severity.test.ts b/sdk/typescript/tests-ts/classify-severity.test.ts
index b972a4aa1..e0488fcdc 100644
--- a/sdk/typescript/tests-ts/classify-severity.test.ts
+++ b/sdk/typescript/tests-ts/classify-severity.test.ts
@@ -1,3 +1,5 @@
+import { defaultSeverityRubric } from "../src/default-severity-rubric.js";
+import { workflowDigest } from "../src/finding-workflow.js";
import { modelResponseText } from "./support/model-response-text.js";
import { writeFile } from "node:fs/promises";
import { join } from "node:path";
@@ -53,7 +55,13 @@ function fakeCodex(response: unknown) {
return {
async run(prompt, turn) {
calls.push({ prompt, thread, turn });
- return { finalResponse: modelResponseText(response) };
+ return {
+ finalResponse: modelResponseText(
+ typeof response === "function"
+ ? response(calls.length)
+ : response,
+ ),
+ };
},
};
},
@@ -61,19 +69,23 @@ function fakeCodex(response: unknown) {
return { codex, calls };
}
-test("without a rubric reuses severity without authentication or a model call", async () => {
- const { codex, calls } = fakeCodex("must not run");
+test("without a rubric runs three independent reviews with the built-in policy", async () => {
+ const { codex, calls } = fakeCodex(assessed);
const original = structuredClone(finding);
const result = await classifySeverity([finding], { codex, environment: {} });
- expect(calls).toHaveLength(0);
- expect(result.rubricSha256).toBeNull();
+ expect(calls).toHaveLength(3);
+ expect(new Set(calls.map((call) => call.thread)).size).toBe(3);
+ expect(JSON.parse(calls[0]!.prompt.split("\n\n").at(-1)!).rubric).toEqual([
+ defaultSeverityRubric,
+ ]);
+ expect(result.rubricSha256).toBe(workflowDigest([defaultSeverityRubric]));
expect(result.assessments[0]).toMatchObject({
findingId: finding.findingId,
occurrenceId: finding.occurrenceId,
decision: "assessed",
- level: "high",
- source: "existing-severity",
- rationale: "Original assessment",
+ level: "medium",
+ source: "rubric",
+ rationale: assessed.rationale,
});
expect(finding).toEqual(original);
expect(
@@ -104,7 +116,13 @@ test("supplies complete evidence and separate policy/context to a restricted str
});
expect(result.rubricSha256).toMatch(/^[a-f0-9]{64}$/u);
expect(result.knowledgeBaseSha256).toMatch(/^[a-f0-9]{64}$/u);
- expect(calls).toHaveLength(1);
+ expect(calls).toHaveLength(3);
+ expect(new Set(calls.map((call) => call.thread)).size).toBe(3);
+ expect(new Set(calls.map((call) => call.prompt)).size).toBe(1);
+ expect(JSON.parse(calls[0]!.prompt.split("\n\n").at(-1)!).rubric).toEqual([
+ "Assign MEDIUM to bounded unauthorized metadata reads.",
+ ]);
+ expect(calls[0]!.prompt).not.toContain(defaultSeverityRubric);
expect(calls[0]!.prompt).toContain(String(finding["evidence"]));
expect(calls[0]!.prompt).toContain("The counters contain no customer data.");
expect(calls[0]!.thread).toMatchObject({
@@ -151,19 +169,15 @@ test("represents policy exclusions independently from Low", async () => {
expect(validateSeverityClassification(result, [finding])).toEqual(result);
});
-test("classifies imported reports without severity when a rubric is supplied", async () => {
+test("classifies imported reports without severity using the default rubric", async () => {
const {
severity: _severity,
occurrenceId: _occurrenceId,
...imported
} = finding;
- await expect(classifySeverity([imported])).rejects.toThrow("supply a rubric");
- const rubricPath = await document(
- "Classify bounded metadata reads as medium.",
- );
const { codex } = fakeCodex(assessed);
expect(
- (await classifySeverity([imported], { rubricPath, codex })).assessments[0]!
+ (await classifySeverity([imported], { codex })).assessments[0]!
.occurrenceId,
).toBeNull();
});
@@ -230,3 +244,104 @@ test("cancellation and invalid inputs cannot produce a successful classification
);
expect((await classifySeverity([])).assessments).toEqual([]);
});
+
+test("votes independently for each imported report and retains the winning answer", async () => {
+ const rubricPath = await document("Classify using the supplied evidence.");
+ const second = { ...finding, findingId: "second-finding" };
+ const responses = [
+ { ...assessed, level: "low", rationale: "First low answer" },
+ { ...assessed, level: "high" },
+ { ...assessed, level: "low", rationale: "Second low answer" },
+ ...["medium", "critical", "high"].map((level) => ({
+ ...assessed,
+ findingId: second.findingId,
+ level,
+ })),
+ ];
+ const { codex, calls } = fakeCodex((call: number) => responses[call - 1]);
+ const result = await classifySeverity([finding, second], {
+ rubricPath,
+ codex,
+ });
+ expect(calls).toHaveLength(6);
+ expect(result.assessments).toMatchObject([
+ { level: "low", rationale: "First low answer" },
+ { level: "critical", findingId: second.findingId },
+ ]);
+ expect(
+ calls.map(
+ ({ prompt }) =>
+ JSON.parse(prompt.split("\n\n").at(-1)!).finding.findingId,
+ ),
+ ).toEqual([
+ finding.findingId,
+ finding.findingId,
+ finding.findingId,
+ second.findingId,
+ second.findingId,
+ second.findingId,
+ ]);
+});
+
+test.each([2, 3, 10])(
+ "does not return a partial vote when review %s fails or is invalid",
+ async (failedRun) => {
+ for (const invalid of [false, true]) {
+ const { codex, calls } = fakeCodex((call: number) => {
+ if (call !== failedRun) return assessed;
+ if (invalid) return { ...assessed, findingId: "wrong-finding" };
+ throw new Error("Review failed");
+ });
+ await expect(
+ classifySeverity([finding], {
+ codex,
+ classificationRuns: Math.max(3, failedRun),
+ }),
+ ).rejects.toThrow(invalid ? "invalid assessment" : "Review failed");
+ expect(calls).toHaveLength(failedRun);
+ }
+ },
+);
+
+test.each([1, 2, 3])(
+ "cancellation after review %s stops classification",
+ async (canceledRun) => {
+ const controller = new AbortController();
+ const { codex, calls } = fakeCodex((call: number) => {
+ if (call === canceledRun) controller.abort(new Error("Stopped"));
+ return assessed;
+ });
+ await expect(
+ classifySeverity([finding], {
+ codex,
+ signal: controller.signal,
+ }),
+ ).rejects.toThrow("Stopped");
+ expect(calls).toHaveLength(canceledRun);
+ },
+);
+
+test.each([1, 2, 4, 10])(
+ "runs %i independent local reviews",
+ async (classificationRuns) => {
+ const { codex, calls } = fakeCodex(assessed);
+ const result = await classifySeverity([finding], {
+ codex,
+ classificationRuns,
+ });
+ expect(calls).toHaveLength(classificationRuns);
+ expect(result.classificationRuns).toBe(classificationRuns);
+ expect(result.assessments[0]!.level).toBe("medium");
+ },
+);
+
+test.each([0, -1, 11, 1.5, NaN, Infinity])(
+ "rejects invalid local run count %s before model work",
+ async (classificationRuns) => {
+ const { codex, calls } = fakeCodex(assessed);
+ await expect(
+ classifySeverity([finding], { codex, classificationRuns }),
+ ).rejects.toThrow();
+ expect(calls).toHaveLength(0);
+ },
+);
diff --git a/sdk/typescript/tests-ts/cli-classify-severity.test.ts b/sdk/typescript/tests-ts/cli-classify-severity.test.ts
index 262064afb..0026574ec 100644
--- a/sdk/typescript/tests-ts/cli-classify-severity.test.ts
+++ b/sdk/typescript/tests-ts/cli-classify-severity.test.ts
@@ -67,7 +67,7 @@ test.each([
},
);
-test("classify-severity accepts external scan directories and defaults to existing severity", async () => {
+test("classify-severity accepts external scan directories and uses the default rubric when none is supplied", async () => {
const deps = dependencies();
let called = false;
deps.classifyScanDirectorySeverity = async (directory, options, surface) => {
@@ -75,6 +75,7 @@ test("classify-severity accepts external scan directories and defaults to existi
expect(directory).toBe(resolve(deps.currentDirectory(), "saved scan"));
expect(options!.rubricPath).toBeUndefined();
expect(options!.reprocess).toBe(false);
+ expect(options!.classificationRuns).toBe(3);
expect(options!.findingIds).toBeUndefined();
expect(surface).toBe("cli");
return result;
@@ -195,3 +196,45 @@ test("publication forwards selected finding IDs only to Linear", async () => {
),
).toBe(2);
});
+
+test.each([1, 2, 10])("forwards --runs %i", async (classificationRuns) => {
+ const deps = dependencies();
+ const classify = mock(async (_scan: string, options: any) => {
+ expect(options.classificationRuns).toBe(classificationRuns);
+ return result;
+ });
+ deps.classifyScanSeverity = classify;
+ expect(
+ await runCapturedCli(
+ main,
+ [
+ "classify-severity",
+ "--scan",
+ "latest",
+ "--runs",
+ String(classificationRuns),
+ ],
+ deps,
+ ),
+ ).toBe(0);
+ expect(classify).toHaveBeenCalledTimes(1);
+});
+
+test.each(["0", "-1", "11", "1.5", "NaN", "Infinity", "invalid"])(
+ "rejects --runs %s in either input mode",
+ async (value) => {
+ const deps = dependencies();
+ const classify = mock(async () => result);
+ deps.classifyScanSeverity = classify;
+ for (const mode of [["--scan", "latest"], ["--records"]]) {
+ expect(
+ await runCapturedCli(
+ main,
+ ["classify-severity", ...mode, "--runs", value],
+ deps,
+ ),
+ ).toBe(2);
+ }
+ expect(classify).toHaveBeenCalledTimes(0);
+ },
+);
diff --git a/sdk/typescript/tests-ts/severity-review.test.ts b/sdk/typescript/tests-ts/severity-review.test.ts
new file mode 100644
index 000000000..c7f3496f6
--- /dev/null
+++ b/sdk/typescript/tests-ts/severity-review.test.ts
@@ -0,0 +1,77 @@
+import { expect, test } from "bun:test";
+import {
+ reviewSeverity,
+ type SeverityDecision,
+} from "../src/severity-review.js";
+
+// Listed from lowest to highest independently of the runtime ordering.
+const levels = [
+ null,
+ "informational",
+ "low",
+ "medium",
+ "high",
+ "critical",
+] as const;
+
+test("three independent reviews vote by normalized severity, with deterministic ties", async () => {
+ for (const first of levels) {
+ for (const second of levels) {
+ for (const third of levels) {
+ const votes = [first, second, third] as const;
+ const answers: SeverityDecision[] = votes.map((level, index) => ({
+ findingId: "example-finding",
+ decision: level === null ? "excluded" : "assessed",
+ level,
+ rubricLabel: level === null ? null : `${level}-${index}`,
+ rationale: `Review ${index + 1}`,
+ confidence: index === 0 ? "low" : "high",
+ reviewTrigger: `Missing fact ${index + 1}`,
+ }));
+ let calls = 0;
+ const result = await reviewSeverity(async () => answers[calls++]!);
+ const expected =
+ first === second || first === third
+ ? first
+ : second === third
+ ? second
+ : levels[
+ Math.max(...votes.map((level) => levels.indexOf(level)))
+ ];
+ expect(calls).toBe(3);
+ expect(result).toBe(answers.find(({ level }) => level === expected)!);
+ }
+ }
+ }
+});
+
+test.each([
+ [["low"], "low"],
+ [["low", "high"], "high"],
+ [["low", "low", "high", "high"], "high"],
+ [["critical", "low", "low", "high", "high", "informational"], "high"],
+ [[null, null, "low", "low"], "low"],
+ [
+ ["low", "low", "low", "low", "low", "high", "high", "high", "high", "high"],
+ "high",
+ ],
+] as [SeverityDecision["level"][], SeverityDecision["level"]][])(
+ "votes across %j",
+ async (votes, expected) => {
+ let calls = 0;
+ const result = await reviewSeverity(async () => {
+ const level = votes[calls++]!;
+ return {
+ findingId: "example",
+ decision: level === null ? "excluded" : "assessed",
+ level,
+ rubricLabel: level,
+ rationale: "Synthetic review",
+ confidence: null,
+ reviewTrigger: null,
+ };
+ }, votes.length);
+ expect(calls).toBe(votes.length);
+ expect(result.level).toBe(expected);
+ },
+);