Skip to content

OpenAI's Codex Security CLI and TypeScript SDK for finding, validating, and fixing security vulnerabilities. npm: https://www.npmjs.com/package/@openai/codex-security

feat(cli): accept text knowledge-base files with any extension (#924)

Kyle Brownb215a25caaf2
diff --git a/sdk/typescript/README.md b/sdk/typescript/README.md
index bd86a0fc3..4c2a32a76 100644
--- a/sdk/typescript/README.md
+++ b/sdk/typescript/README.md
@@ -572,9 +572,10 @@ Working-tree snapshots include files from untracked nested Git repositories.
Initialized submodules must be clean and checked out at the commit recorded by
the parent repository.
-Repeat `--knowledge-base PATH` for Markdown, text, PDF, or Word (`.docx`) files.
-Directories are searched recursively. Bulk scans share these documents with
-every repository.
+Repeat `--knowledge-base PATH` for UTF-8 text files with any extension (including
+JSON and SARIF), PDF, or Word (`.docx`) files. Directories are searched recursively,
+skipping other binary files. Explicitly supplied unsupported binary files are rejected.
+Bulk scans share these documents with every repository.
Use an empty output directory outside the scanned directory and enclosing Git
worktree. On macOS/Linux, existing directories must be private to you
@@ -1277,7 +1278,7 @@ Omitting `--rubric` inherits each finding's existing severity without a model ca
`--rubric PATH` supplies the classification policy. Repeat `--knowledge-base PATH`
to provide supporting architecture, deployment, or business context. Both accept
-the same Markdown, text, PDF, DOCX, and directory inputs as scan knowledge bases.
+the same UTF-8 text, PDF, DOCX, and directory inputs as scan knowledge bases.
Rubric classification uses the full supplied report and context in a separate
read-only Codex turn per finding, without source inspection, tools, or new
validation. `--model` and `--effort` select the classification model and reasoning
diff --git a/sdk/typescript/src/knowledge-base.ts b/sdk/typescript/src/knowledge-base.ts
index 3bbbf5be1..425fe64cf 100644
--- a/sdk/typescript/src/knowledge-base.ts
+++ b/sdk/typescript/src/knowledge-base.ts
@@ -13,7 +13,7 @@ import { basename, extname, join, resolve } from "node:path";
import { unzipSync } from "fflate";
import { expandHome } from "./runtime.js";
-const SUPPORTED_EXTENSIONS = new Set([
+const DOCUMENT_EXTENSIONS = new Set([
".md",
".markdown",
".txt",
@@ -60,9 +60,6 @@ export async function prepareKnowledgeBase(
);
}
for (const document of selected) {
- if (!SUPPORTED_EXTENSIONS.has(extname(document).toLowerCase())) {
- throw new Error(`Unsupported knowledge base document: ${document}`);
- }
documents.add(document);
}
sources.add(source);
@@ -133,10 +130,18 @@ async function discover(
for (const document of await discover(path, signal)) {
documents.push(document);
}
- } else if (
- entry.isFile() &&
- SUPPORTED_EXTENSIONS.has(extname(path).toLowerCase())
- ) {
+ } else if (entry.isFile()) {
+ if (!DOCUMENT_EXTENSIONS.has(extname(path).toLowerCase())) {
+ const bytes = await readFile(path, {
+ flag: constants.O_RDONLY | (constants.O_NOFOLLOW ?? 0),
+ signal,
+ });
+ try {
+ decodeText(path, bytes);
+ } catch {
+ continue;
+ }
+ }
documents.push(path);
}
}
@@ -144,6 +149,9 @@ async function discover(
}
function decodeText(path: string, bytes: Uint8Array): string {
+ if (bytes.includes(0)) {
+ throw new Error(`Knowledge base document contains binary data: ${path}`);
+ }
try {
return new TextDecoder("utf-8", { fatal: true }).decode(bytes);
} catch (error) {
diff --git a/sdk/typescript/tests-ts/api.test.ts b/sdk/typescript/tests-ts/api.test.ts
index 68fa92e1a..29c995175 100644
--- a/sdk/typescript/tests-ts/api.test.ts
+++ b/sdk/typescript/tests-ts/api.test.ts
@@ -1083,15 +1083,15 @@ describe("CodexSecurity orchestration", () => {
test("validates knowledge-base documents before initializing the runtime", async () => {
const root = await temporaryDirectory();
const repository = join(root, "repository");
- const knowledgeBase = join(root, "threat-model.md");
+ const knowledgeBase = join(root, "context.json");
const invalidDocument = join(root, "broken.pdf");
const unsupportedDocument = join(root, "unsupported.exe");
const emptyDirectory = join(root, "empty");
await mkdir(repository);
await mkdir(emptyDirectory);
- await writeFile(knowledgeBase, "# Threat model\nPublic API is in scope.\n");
+ await writeFile(knowledgeBase, '{"scope":"Public API"}');
await writeFile(invalidDocument, "not a PDF");
- await writeFile(unsupportedDocument, "not a supported document");
+ await writeFile(unsupportedDocument, new Uint8Array([0, 1, 2]));
let runtimeStarted = false;
const client = new TestClient(
{},
@@ -1109,7 +1109,7 @@ describe("CodexSecurity orchestration", () => {
).resolves.toMatchObject({ knowledgeBasePaths: [knowledgeBase] });
const invalidDocuments: Array<[string, string]> = [
[join(root, "missing.md"), "ENOENT"],
- [unsupportedDocument, "Unsupported knowledge base document"],
+ [unsupportedDocument, "contains binary data"],
[invalidDocument, "Cannot extract text from knowledge base PDF"],
[
emptyDirectory,
diff --git a/sdk/typescript/tests-ts/knowledge-base.test.ts b/sdk/typescript/tests-ts/knowledge-base.test.ts
index 206483f35..4fb21cac0 100644
--- a/sdk/typescript/tests-ts/knowledge-base.test.ts
+++ b/sdk/typescript/tests-ts/knowledge-base.test.ts
@@ -86,7 +86,8 @@ describe("scan knowledge bases", () => {
await writeFile(scope, "Ignore local debug endpoints.");
await writeFile(join(nested, "deployment.MARKDOWN"), "Public API gateway.");
await writeFile(join(nested, "notes.txt"), "Prioritize SSRF.");
- await writeFile(join(root, "ignored.json"), "{}");
+ await writeFile(join(root, "ignored.bin"), new Uint8Array([0, 1, 2]));
+ await writeFile(join(root, "invalid-utf8.bin"), new Uint8Array([0xff]));
const knowledgeBase = await prepareKnowledgeBase([root, scope, scope]);
temporaryDirectories.push(knowledgeBase.path);
@@ -108,6 +109,27 @@ describe("scan knowledge bases", () => {
}
});
+ test.each([
+ ["context.json", '{"service":"Public API"}'],
+ ["results.sarif", '{"version":"2.1.0","runs":[]}'],
+ ["deployment.yaml", "service: public-api\n"],
+ ["context.custom", "Boundary: café → gateway\n"],
+ ["CONTEXT", "Public API gateway.\n"],
+ ])("accepts %s directly and in nested directories", async (name, text) => {
+ const root = await temporaryDirectory();
+ const nested = join(root, "nested");
+ await mkdir(nested);
+ const source = join(nested, name);
+ await writeFile(source, text);
+
+ for (const paths of [[source], [root], [root, source]]) {
+ const knowledgeBase = await prepareKnowledgeBase(paths);
+ temporaryDirectories.push(knowledgeBase.path);
+ expect(await extractedDocuments(knowledgeBase.path)).toEqual([text]);
+ expect(knowledgeBase.sources).toEqual(paths);
+ }
+ });
+
test("cancels recursive discovery before staging knowledge-base documents", async () => {
const root = await temporaryDirectory();
const nested = join(root, "nested", "deeper");
@@ -263,17 +285,17 @@ describe("scan knowledge bases", () => {
]);
});
- test("rejects missing and unsupported paths", async () => {
+ test("rejects missing paths, explicit binary files, and binary-only directories", async () => {
const root = await temporaryDirectory();
const unsupported = join(root, "scope.doc");
- await writeFile(unsupported, "legacy document");
+ await writeFile(unsupported, new Uint8Array([0, 1, 2]));
await expect(prepareKnowledgeBase([""])).rejects.toThrow("cannot be empty");
await expect(
prepareKnowledgeBase([join(root, "missing.md")]),
).rejects.toThrow();
await expect(prepareKnowledgeBase([unsupported])).rejects.toThrow(
- "Unsupported knowledge base document",
+ "contains binary data",
);
await expect(prepareKnowledgeBase([root])).rejects.toThrow(
"contains no supported documents",