Skip to content

OpenAI's Codex Security CLI and TypeScript SDK for finding, validating, and fixing security vulnerabilities. npm: https://www.npmjs.com/package/@openai/codex-security

feat(sdk): opt in to the remote rubric for standalone reclassification (#2257)

Aismit8ee56c5b7b36
diff --git a/sdk/typescript/docs/cli.md b/sdk/typescript/docs/cli.md
index 8227cf17c..3622cb1c6 100644
--- a/sdk/typescript/docs/cli.md
+++ b/sdk/typescript/docs/cli.md
@@ -1696,12 +1696,19 @@ The [records protocol](classify-severity-records.md) accepts immutable Markdown
report snapshots and delegates model execution to the host.
For saved findings, use exactly one of `--scan` (ID, prefix, or `latest`) and `--scan-dir`.
-Omitting `--rubric` runs classification with the [built-in severity rubric](https://github.com/openai/codex-security/blob/main/sdk/typescript/src/default-severity-rubric.ts).
+By default, saved findings are classified with the [built-in severity rubric](https://github.com/openai/codex-security/blob/main/sdk/typescript/src/default-severity-rubric.ts).
It evaluates attacker access, boundary crossing, additional harm, and evidence for
Critical, High, Medium, Low, and Informational findings. Confidence remains separate
-from severity; missing proof alone does not justify a downgrade. An explicit
-`--rubric` replaces the default policy completely. Rubrics and repeatable
-`--knowledge-base` context accept text, PDF, DOCX, and directories.
+from severity; missing proof alone does not justify a downgrade.
+
+Set `CODEX_SECURITY_ENABLE_REMOTE_SEVERITY_RUBRIC=1` to fetch the current default
+from Statsig once per reclassification. Only the exact value `1` enables the
+request. It sends the config name without findings or account identifiers.
+If the request fails, times out, or returns an invalid rubric, the built-in
+rubric is used. An explicit `--rubric` (SDK `rubricPath`) skips the lookup and
+uses your policy for every review. Rubrics and repeatable `--knowledge-base`
+context accept text, PDF, DOCX, and directories.
+
Each finding is classified three times by default. `--runs N` accepts
an integer from 1 through 10; SDK options use `classificationRuns` with the same
default and bounds. Each review runs in an independent read-only turn using the
@@ -1805,7 +1812,7 @@ await publishScan(scanDirectory, {
`classifySeverity()` accepts `findingId`, `title`, `summary`, and available
evidence/metadata. Imported reports may lack severity or occurrence IDs; omitting
-`rubricPath` uses the built-in policy even when a scanner severity is present.
+`rubricPath` uses the selected default policy even when a scanner severity is present.
Scan wrappers persist checkpoints (including for
external directories), accept `reprocess: true`, and treat `findingIds: []` as an
empty selection. Use the same `CODEX_SECURITY_STATE_DIR` for classification and
diff --git a/sdk/typescript/src/classify-severity.ts b/sdk/typescript/src/classify-severity.ts
index 2e65ffdab..7672fcc81 100644
--- a/sdk/typescript/src/classify-severity.ts
+++ b/sdk/typescript/src/classify-severity.ts
@@ -35,7 +35,7 @@ export interface ClassifySeverityOptions extends Pick<
| "signal"
| "workingDirectory"
> {
- /** Classification policy override. Omit to use the built-in severity rubric. */
+ /** Overrides both the bundled default and the optional remote default. */
rubricPath?: string;
/** Independent reviews per finding: integer 1–10, default 3. */
classificationRuns?: number;
@@ -76,6 +76,20 @@ const textSchema = z
.min(1)
.refine((value) => value.trim().length > 0);
const digestSchema = z.string().regex(/^[a-f0-9]{64}$/u);
+const configName = "codex_security_default_severity_rubric";
+const remoteRubricConfig = z.object({
+ name: z.literal(configName),
+ value: z.strictObject({
+ markdown: textSchema.refine(
+ (text) =>
+ Buffer.byteLength(text, "utf8") <= 196_608 &&
+ !/[\u0000-\u0008\u000b\u000c\u000e-\u001f\u007f-\u009f\ud800-\udfff]/u.test(
+ text,
+ ),
+ ),
+ revision: z.number().int().positive(),
+ }),
+});
/** @internal */
export const severityClassificationSchema = z.strictObject({
schemaVersion: z.literal(1),
@@ -122,7 +136,7 @@ export async function classifySeverityInternal(
}
const rubric =
options.rubricPath === undefined
- ? [defaultSeverityRubric]
+ ? [await resolveDefaultRubric(options)]
: await readDocuments([options.rubricPath], options.signal);
const knowledge = options.knowledgeBasePaths?.length
? await readDocuments(options.knowledgeBasePaths, options.signal)
@@ -187,6 +201,41 @@ export async function classifySeverityInternal(
return result;
}
+async function resolveDefaultRubric({
+ environment = process.env,
+ signal,
+}: ClassifySeverityOptions): Promise<string> {
+ if (environment["CODEX_SECURITY_ENABLE_REMOTE_SEVERITY_RUBRIC"] !== "1")
+ return defaultSeverityRubric;
+
+ const timeout = AbortSignal.timeout(3_000);
+ try {
+ const response = await fetch("https://ab.chatgpt.com/v1/get_config", {
+ method: "POST",
+ headers: {
+ "Content-Type": "application/json",
+ // Public client key from openai/codex, codex-rs/otel/src/config.rs.
+ "statsig-api-key": "client-MkRuleRQBd6qakfnDYqJVR9JuXcY57Ljly3vi5JVUIO",
+ },
+ body: JSON.stringify({
+ configName,
+ statsigMetadata: { exposureLoggingDisabled: true },
+ }),
+ redirect: "error",
+ signal: signal ? AbortSignal.any([signal, timeout]) : timeout,
+ });
+ if (!response.ok) {
+ await response.body?.cancel();
+ return defaultSeverityRubric;
+ }
+ return remoteRubricConfig.parse(await response.json()).value.markdown;
+ } catch {
+ return defaultSeverityRubric;
+ } finally {
+ signal?.throwIfAborted();
+ }
+}
+
async function readDocuments(
paths: readonly string[],
signal?: AbortSignal,
diff --git a/sdk/typescript/src/cli.ts b/sdk/typescript/src/cli.ts
index bddc23e63..b9093dad8 100644
--- a/sdk/typescript/src/cli.ts
+++ b/sdk/typescript/src/cli.ts
@@ -4955,7 +4955,7 @@ export async function main(
})
.command("classify-severity", {
description: "Assess severity for saved findings or canonical records.",
- hint: "Use --rubric to override the built-in policy or --records for host-provided classification.",
+ hint: "For saved findings, use --rubric for your own policy or set CODEX_SECURITY_ENABLE_REMOTE_SEVERITY_RUBRIC=1 for the Statsig default. For host-provided reviews, use --records.",
destructive: true,
mcp: false,
options: z.object({
@@ -4985,7 +4985,7 @@ export async function main(
rubric: optionValue("--rubric")
.optional()
.describe(
- "Classification policy document; omit to classify with the built-in severity rubric.",
+ "Classification policy document. Takes precedence over the default rubric.",
),
knowledgeBase: z
.array(optionValue("--knowledge-base"))
diff --git a/sdk/typescript/tests-ts/classify-severity.test.ts b/sdk/typescript/tests-ts/classify-severity.test.ts
index 90d7a8b63..638751e79 100644
--- a/sdk/typescript/tests-ts/classify-severity.test.ts
+++ b/sdk/typescript/tests-ts/classify-severity.test.ts
@@ -5,7 +5,7 @@ import { writeFile } from "node:fs/promises";
import { join } from "node:path";
import type { ThreadOptions, TurnOptions } from "@openai/codex-sdk";
import Ajv2020 from "ajv/dist/2020.js";
-import { afterEach, expect, test } from "bun:test";
+import { afterEach, expect, spyOn, test } from "bun:test";
import {
classifySeverity,
validateSeverityClassification,
@@ -47,6 +47,12 @@ const assessed = {
reviewTrigger: "Protected content in the response would increase severity.",
};
+const optInEnvironment = { CODEX_SECURITY_ENABLE_REMOTE_SEVERITY_RUBRIC: "1" };
+const remoteConfig = {
+ name: "codex_security_default_severity_rubric",
+ value: { markdown: "Apply this updated severity rubric.", revision: 2 },
+};
+
function fakeCodex(response: unknown) {
const calls: { prompt: string; thread: ThreadOptions; turn: TurnOptions }[] =
[];
@@ -69,33 +75,124 @@ function fakeCodex(response: unknown) {
return { codex, calls };
}
-test("without a rubric runs three independent reviews with the built-in policy", async () => {
+test.each([undefined, "0", "true", " 1"])(
+ "uses the bundled rubric without contacting Statsig when the flag is %j",
+ async (value) => {
+ using request = spyOn(globalThis, "fetch").mockResolvedValue(
+ Response.json(remoteConfig),
+ );
+ const { codex, calls } = fakeCodex(assessed);
+ const original = structuredClone(finding);
+ const result = await classifySeverity([finding], {
+ codex,
+ environment: { CODEX_SECURITY_ENABLE_REMOTE_SEVERITY_RUBRIC: value },
+ });
+ expect(request).not.toHaveBeenCalled();
+ expect(calls).toHaveLength(3);
+ expect(new Set(calls.map((call) => call.thread)).size).toBe(3);
+ expect(JSON.parse(calls[0]!.prompt.split("\n\n").at(-1)!).rubric).toEqual([
+ defaultSeverityRubric,
+ ]);
+ expect(result.rubricSha256).toBe(workflowDigest([defaultSeverityRubric]));
+ expect(result.assessments[0]).toMatchObject({
+ findingId: finding.findingId,
+ occurrenceId: finding.occurrenceId,
+ decision: "assessed",
+ level: "medium",
+ source: "rubric",
+ rationale: assessed.rationale,
+ });
+ expect(finding).toEqual(original);
+ expect(
+ validateSeverityClassification(JSON.parse(JSON.stringify(result)), [
+ finding,
+ ]),
+ ).toEqual(result);
+ },
+);
+
+test("opt-in fetches only the named config once for all reviews", async () => {
+ using request = spyOn(globalThis, "fetch").mockResolvedValue(
+ Response.json(remoteConfig),
+ );
const { codex, calls } = fakeCodex(assessed);
- const original = structuredClone(finding);
- const result = await classifySeverity([finding], { codex, environment: {} });
+ const result = await classifySeverity([finding], {
+ codex,
+ environment: optInEnvironment,
+ });
+ expect(request).toHaveBeenCalledTimes(1);
+ const configRequest = new Request(...request.mock.calls[0]!);
+ expect(configRequest.url).toBe("https://ab.chatgpt.com/v1/get_config");
+ expect(configRequest.method).toBe("POST");
+ expect(configRequest.headers.has("authorization")).toBe(false);
+ expect(await configRequest.json()).toEqual({
+ configName: remoteConfig.name,
+ statsigMetadata: { exposureLoggingDisabled: true },
+ });
expect(calls).toHaveLength(3);
- expect(new Set(calls.map((call) => call.thread)).size).toBe(3);
- expect(JSON.parse(calls[0]!.prompt.split("\n\n").at(-1)!).rubric).toEqual([
- defaultSeverityRubric,
- ]);
- expect(result.rubricSha256).toBe(workflowDigest([defaultSeverityRubric]));
- expect(result.assessments[0]).toMatchObject({
- findingId: finding.findingId,
- occurrenceId: finding.occurrenceId,
- decision: "assessed",
- level: "medium",
- source: "rubric",
- rationale: assessed.rationale,
+ for (const { prompt } of calls) {
+ expect(JSON.parse(prompt.split("\n\n").at(-1)!).rubric).toEqual([
+ remoteConfig.value.markdown,
+ ]);
+ }
+ expect(result.rubricSha256).toBe(
+ workflowDigest([remoteConfig.value.markdown]),
+ );
+});
+
+test.each([
+ ["an unavailable service", new Response(null, { status: 503 })],
+ [
+ "a blank policy",
+ Response.json({ ...remoteConfig, value: { markdown: " ", revision: 2 } }),
+ ],
+ ["a network error", new TypeError("Offline")],
+] as const)(
+ "falls back to the bundled rubric on %s",
+ async (_scenario, failure) => {
+ using request = spyOn(globalThis, "fetch");
+ if (failure instanceof Error) request.mockRejectedValue(failure);
+ else request.mockResolvedValue(failure);
+ const { codex, calls } = fakeCodex(assessed);
+ const result = await classifySeverity([finding], {
+ codex,
+ environment: optInEnvironment,
+ classificationRuns: 1,
+ });
+ expect(request).toHaveBeenCalledTimes(1);
+ expect(JSON.parse(calls[0]!.prompt.split("\n\n").at(-1)!).rubric).toEqual([
+ defaultSeverityRubric,
+ ]);
+ expect(result.rubricSha256).toBe(workflowDigest([defaultSeverityRubric]));
+ },
+);
+
+test("cancellation while reading the remote rubric stops the reviews", async () => {
+ const controller = new AbortController();
+ const reason = new Error("Canceled reclassification");
+ const response = Response.json(remoteConfig);
+ using read = spyOn(response, "json").mockImplementation(async () => {
+ controller.abort(reason);
+ return remoteConfig;
});
- expect(finding).toEqual(original);
- expect(
- validateSeverityClassification(JSON.parse(JSON.stringify(result)), [
- finding,
- ]),
- ).toEqual(result);
+ using request = spyOn(globalThis, "fetch").mockResolvedValue(response);
+ const { codex, calls } = fakeCodex(assessed);
+ await expect(
+ classifySeverity([finding], {
+ codex,
+ signal: controller.signal,
+ environment: optInEnvironment,
+ }),
+ ).rejects.toBe(reason);
+ expect(request).toHaveBeenCalledTimes(1);
+ expect(read).toHaveBeenCalledTimes(1);
+ expect(calls).toHaveLength(0);
});
test("supplies complete evidence and separate policy/context to a restricted structured Codex turn", async () => {
+ using request = spyOn(globalThis, "fetch").mockResolvedValue(
+ Response.json(remoteConfig),
+ );
const rubricPath = await document(
"Assign MEDIUM to bounded unauthorized metadata reads.",
);
@@ -103,6 +200,7 @@ test("supplies complete evidence and separate policy/context to a restricted str
const { codex, calls } = fakeCodex(assessed);
const signal = new AbortController().signal;
const result = await classifySeverity([finding], {
+ environment: optInEnvironment,
rubricPath,
knowledgeBasePaths: [knowledge],
codex,
@@ -110,6 +208,7 @@ test("supplies complete evidence and separate policy/context to a restricted str
model: "synthetic-model",
reasoningEffort: "future-effort",
});
+ expect(request).not.toHaveBeenCalled();
expect(result.assessments[0]).toMatchObject({
...assessed,
source: "rubric",