OpenAI's Codex Security CLI and TypeScript SDK for finding, validating, and fixing security vulnerabilities. npm: https://www.npmjs.com/package/@openai/codex-security
TypeScript ai-securityapplication-securityclicode-scanningcodexcodex-securitycybersecuritydevsecopsnodejsnpmopenaisdksecuritytypescriptvulnerability-scanning
feat(sdk): opt in to the remote rubric for standalone reclassification (#2257)
Aismit8ee56c5b7b36
diff --git a/sdk/typescript/docs/cli.md b/sdk/typescript/docs/cli.mdindex 8227cf17c..3622cb1c6 100644--- a/sdk/typescript/docs/cli.md+++ b/sdk/typescript/docs/cli.md@@ -1696,12 +1696,19 @@ The [records protocol](classify-severity-records.md) accepts immutable Markdownreport snapshots and delegates model execution to the host.For saved findings, use exactly one of `--scan` (ID, prefix, or `latest`) and `--scan-dir`.-Omitting `--rubric` runs classification with the [built-in severity rubric](https://github.com/openai/codex-security/blob/main/sdk/typescript/src/default-severity-rubric.ts).+By default, saved findings are classified with the [built-in severity rubric](https://github.com/openai/codex-security/blob/main/sdk/typescript/src/default-severity-rubric.ts).It evaluates attacker access, boundary crossing, additional harm, and evidence forCritical, High, Medium, Low, and Informational findings. Confidence remains separate-from severity; missing proof alone does not justify a downgrade. An explicit-`--rubric` replaces the default policy completely. Rubrics and repeatable-`--knowledge-base` context accept text, PDF, DOCX, and directories.+from severity; missing proof alone does not justify a downgrade.++Set `CODEX_SECURITY_ENABLE_REMOTE_SEVERITY_RUBRIC=1` to fetch the current default+from Statsig once per reclassification. Only the exact value `1` enables the+request. It sends the config name without findings or account identifiers.+If the request fails, times out, or returns an invalid rubric, the built-in+rubric is used. An explicit `--rubric` (SDK `rubricPath`) skips the lookup and+uses your policy for every review. Rubrics and repeatable `--knowledge-base`+context accept text, PDF, DOCX, and directories.+Each finding is classified three times by default. `--runs N` acceptsan integer from 1 through 10; SDK options use `classificationRuns` with the samedefault and bounds. Each review runs in an independent read-only turn using the@@ -1805,7 +1812,7 @@ await publishScan(scanDirectory, {`classifySeverity()` accepts `findingId`, `title`, `summary`, and availableevidence/metadata. Imported reports may lack severity or occurrence IDs; omitting-`rubricPath` uses the built-in policy even when a scanner severity is present.+`rubricPath` uses the selected default policy even when a scanner severity is present.Scan wrappers persist checkpoints (including forexternal directories), accept `reprocess: true`, and treat `findingIds: []` as anempty selection. Use the same `CODEX_SECURITY_STATE_DIR` for classification anddiff --git a/sdk/typescript/src/classify-severity.ts b/sdk/typescript/src/classify-severity.tsindex 2e65ffdab..7672fcc81 100644--- a/sdk/typescript/src/classify-severity.ts+++ b/sdk/typescript/src/classify-severity.ts@@ -35,7 +35,7 @@ export interface ClassifySeverityOptions extends Pick<| "signal"| "workingDirectory"> {- /** Classification policy override. Omit to use the built-in severity rubric. */+ /** Overrides both the bundled default and the optional remote default. */rubricPath?: string;/** Independent reviews per finding: integer 1–10, default 3. */classificationRuns?: number;@@ -76,6 +76,20 @@ const textSchema = z.min(1).refine((value) => value.trim().length > 0);const digestSchema = z.string().regex(/^[a-f0-9]{64}$/u);+const configName = "codex_security_default_severity_rubric";+const remoteRubricConfig = z.object({+ name: z.literal(configName),+ value: z.strictObject({+ markdown: textSchema.refine(+ (text) =>+ Buffer.byteLength(text, "utf8") <= 196_608 &&+ !/[\u0000-\u0008\u000b\u000c\u000e-\u001f\u007f-\u009f\ud800-\udfff]/u.test(+ text,+ ),+ ),+ revision: z.number().int().positive(),+ }),+});/** @internal */export const severityClassificationSchema = z.strictObject({schemaVersion: z.literal(1),@@ -122,7 +136,7 @@ export async function classifySeverityInternal(}const rubric =options.rubricPath === undefined- ? [defaultSeverityRubric]+ ? [await resolveDefaultRubric(options)]: await readDocuments([options.rubricPath], options.signal);const knowledge = options.knowledgeBasePaths?.length? await readDocuments(options.knowledgeBasePaths, options.signal)@@ -187,6 +201,41 @@ export async function classifySeverityInternal(return result;}+async function resolveDefaultRubric({+ environment = process.env,+ signal,+}: ClassifySeverityOptions): Promise<string> {+ if (environment["CODEX_SECURITY_ENABLE_REMOTE_SEVERITY_RUBRIC"] !== "1")+ return defaultSeverityRubric;++ const timeout = AbortSignal.timeout(3_000);+ try {+ const response = await fetch("https://ab.chatgpt.com/v1/get_config", {+ method: "POST",+ headers: {+ "Content-Type": "application/json",+ // Public client key from openai/codex, codex-rs/otel/src/config.rs.+ "statsig-api-key": "client-MkRuleRQBd6qakfnDYqJVR9JuXcY57Ljly3vi5JVUIO",+ },+ body: JSON.stringify({+ configName,+ statsigMetadata: { exposureLoggingDisabled: true },+ }),+ redirect: "error",+ signal: signal ? AbortSignal.any([signal, timeout]) : timeout,+ });+ if (!response.ok) {+ await response.body?.cancel();+ return defaultSeverityRubric;+ }+ return remoteRubricConfig.parse(await response.json()).value.markdown;+ } catch {+ return defaultSeverityRubric;+ } finally {+ signal?.throwIfAborted();+ }+}+async function readDocuments(paths: readonly string[],signal?: AbortSignal,diff --git a/sdk/typescript/src/cli.ts b/sdk/typescript/src/cli.tsindex bddc23e63..b9093dad8 100644--- a/sdk/typescript/src/cli.ts+++ b/sdk/typescript/src/cli.ts@@ -4955,7 +4955,7 @@ export async function main(}).command("classify-severity", {description: "Assess severity for saved findings or canonical records.",- hint: "Use --rubric to override the built-in policy or --records for host-provided classification.",+ hint: "For saved findings, use --rubric for your own policy or set CODEX_SECURITY_ENABLE_REMOTE_SEVERITY_RUBRIC=1 for the Statsig default. For host-provided reviews, use --records.",destructive: true,mcp: false,options: z.object({@@ -4985,7 +4985,7 @@ export async function main(rubric: optionValue("--rubric").optional().describe(- "Classification policy document; omit to classify with the built-in severity rubric.",+ "Classification policy document. Takes precedence over the default rubric.",),knowledgeBase: z.array(optionValue("--knowledge-base"))diff --git a/sdk/typescript/tests-ts/classify-severity.test.ts b/sdk/typescript/tests-ts/classify-severity.test.tsindex 90d7a8b63..638751e79 100644--- a/sdk/typescript/tests-ts/classify-severity.test.ts+++ b/sdk/typescript/tests-ts/classify-severity.test.ts@@ -5,7 +5,7 @@ import { writeFile } from "node:fs/promises";import { join } from "node:path";import type { ThreadOptions, TurnOptions } from "@openai/codex-sdk";import Ajv2020 from "ajv/dist/2020.js";-import { afterEach, expect, test } from "bun:test";+import { afterEach, expect, spyOn, test } from "bun:test";import {classifySeverity,validateSeverityClassification,@@ -47,6 +47,12 @@ const assessed = {reviewTrigger: "Protected content in the response would increase severity.",};+const optInEnvironment = { CODEX_SECURITY_ENABLE_REMOTE_SEVERITY_RUBRIC: "1" };+const remoteConfig = {+ name: "codex_security_default_severity_rubric",+ value: { markdown: "Apply this updated severity rubric.", revision: 2 },+};+function fakeCodex(response: unknown) {const calls: { prompt: string; thread: ThreadOptions; turn: TurnOptions }[] =[];@@ -69,33 +75,124 @@ function fakeCodex(response: unknown) {return { codex, calls };}-test("without a rubric runs three independent reviews with the built-in policy", async () => {+test.each([undefined, "0", "true", " 1"])(+ "uses the bundled rubric without contacting Statsig when the flag is %j",+ async (value) => {+ using request = spyOn(globalThis, "fetch").mockResolvedValue(+ Response.json(remoteConfig),+ );+ const { codex, calls } = fakeCodex(assessed);+ const original = structuredClone(finding);+ const result = await classifySeverity([finding], {+ codex,+ environment: { CODEX_SECURITY_ENABLE_REMOTE_SEVERITY_RUBRIC: value },+ });+ expect(request).not.toHaveBeenCalled();+ expect(calls).toHaveLength(3);+ expect(new Set(calls.map((call) => call.thread)).size).toBe(3);+ expect(JSON.parse(calls[0]!.prompt.split("\n\n").at(-1)!).rubric).toEqual([+ defaultSeverityRubric,+ ]);+ expect(result.rubricSha256).toBe(workflowDigest([defaultSeverityRubric]));+ expect(result.assessments[0]).toMatchObject({+ findingId: finding.findingId,+ occurrenceId: finding.occurrenceId,+ decision: "assessed",+ level: "medium",+ source: "rubric",+ rationale: assessed.rationale,+ });+ expect(finding).toEqual(original);+ expect(+ validateSeverityClassification(JSON.parse(JSON.stringify(result)), [+ finding,+ ]),+ ).toEqual(result);+ },+);++test("opt-in fetches only the named config once for all reviews", async () => {+ using request = spyOn(globalThis, "fetch").mockResolvedValue(+ Response.json(remoteConfig),+ );const { codex, calls } = fakeCodex(assessed);- const original = structuredClone(finding);- const result = await classifySeverity([finding], { codex, environment: {} });+ const result = await classifySeverity([finding], {+ codex,+ environment: optInEnvironment,+ });+ expect(request).toHaveBeenCalledTimes(1);+ const configRequest = new Request(...request.mock.calls[0]!);+ expect(configRequest.url).toBe("https://ab.chatgpt.com/v1/get_config");+ expect(configRequest.method).toBe("POST");+ expect(configRequest.headers.has("authorization")).toBe(false);+ expect(await configRequest.json()).toEqual({+ configName: remoteConfig.name,+ statsigMetadata: { exposureLoggingDisabled: true },+ });expect(calls).toHaveLength(3);- expect(new Set(calls.map((call) => call.thread)).size).toBe(3);- expect(JSON.parse(calls[0]!.prompt.split("\n\n").at(-1)!).rubric).toEqual([- defaultSeverityRubric,- ]);- expect(result.rubricSha256).toBe(workflowDigest([defaultSeverityRubric]));- expect(result.assessments[0]).toMatchObject({- findingId: finding.findingId,- occurrenceId: finding.occurrenceId,- decision: "assessed",- level: "medium",- source: "rubric",- rationale: assessed.rationale,+ for (const { prompt } of calls) {+ expect(JSON.parse(prompt.split("\n\n").at(-1)!).rubric).toEqual([+ remoteConfig.value.markdown,+ ]);+ }+ expect(result.rubricSha256).toBe(+ workflowDigest([remoteConfig.value.markdown]),+ );+});++test.each([+ ["an unavailable service", new Response(null, { status: 503 })],+ [+ "a blank policy",+ Response.json({ ...remoteConfig, value: { markdown: " ", revision: 2 } }),+ ],+ ["a network error", new TypeError("Offline")],+] as const)(+ "falls back to the bundled rubric on %s",+ async (_scenario, failure) => {+ using request = spyOn(globalThis, "fetch");+ if (failure instanceof Error) request.mockRejectedValue(failure);+ else request.mockResolvedValue(failure);+ const { codex, calls } = fakeCodex(assessed);+ const result = await classifySeverity([finding], {+ codex,+ environment: optInEnvironment,+ classificationRuns: 1,+ });+ expect(request).toHaveBeenCalledTimes(1);+ expect(JSON.parse(calls[0]!.prompt.split("\n\n").at(-1)!).rubric).toEqual([+ defaultSeverityRubric,+ ]);+ expect(result.rubricSha256).toBe(workflowDigest([defaultSeverityRubric]));+ },+);++test("cancellation while reading the remote rubric stops the reviews", async () => {+ const controller = new AbortController();+ const reason = new Error("Canceled reclassification");+ const response = Response.json(remoteConfig);+ using read = spyOn(response, "json").mockImplementation(async () => {+ controller.abort(reason);+ return remoteConfig;});- expect(finding).toEqual(original);- expect(- validateSeverityClassification(JSON.parse(JSON.stringify(result)), [- finding,- ]),- ).toEqual(result);+ using request = spyOn(globalThis, "fetch").mockResolvedValue(response);+ const { codex, calls } = fakeCodex(assessed);+ await expect(+ classifySeverity([finding], {+ codex,+ signal: controller.signal,+ environment: optInEnvironment,+ }),+ ).rejects.toBe(reason);+ expect(request).toHaveBeenCalledTimes(1);+ expect(read).toHaveBeenCalledTimes(1);+ expect(calls).toHaveLength(0);});test("supplies complete evidence and separate policy/context to a restricted structured Codex turn", async () => {+ using request = spyOn(globalThis, "fetch").mockResolvedValue(+ Response.json(remoteConfig),+ );const rubricPath = await document("Assign MEDIUM to bounded unauthorized metadata reads.",);@@ -103,6 +200,7 @@ test("supplies complete evidence and separate policy/context to a restricted strconst { codex, calls } = fakeCodex(assessed);const signal = new AbortController().signal;const result = await classifySeverity([finding], {+ environment: optInEnvironment,rubricPath,knowledgeBasePaths: [knowledge],codex,@@ -110,6 +208,7 @@ test("supplies complete evidence and separate policy/context to a restricted strmodel: "synthetic-model",reasoningEffort: "future-effort",});+ expect(request).not.toHaveBeenCalled();expect(result.assessments[0]).toMatchObject({...assessed,source: "rubric",