OpenAI's Codex Security CLI and TypeScript SDK for finding, validating, and fixing security vulnerabilities. npm: https://www.npmjs.com/package/@openai/codex-security
TypeScript ai-securityapplication-securityclicode-scanningcodexcodex-securitycybersecuritydevsecopsnodejsnpmopenaisdksecuritytypescriptvulnerability-scanning
fix(knowledge-base): drop deleted revisions and field codes from DOCX (#1714)
12c8a7e3bddc67
diff --git a/sdk/typescript/package.json b/sdk/typescript/package.jsonindex 5ff1ed45b..4583a17ae 100644--- a/sdk/typescript/package.json+++ b/sdk/typescript/package.json@@ -90,6 +90,7 @@"papaparse": "5.7.0","pdfjs-dist": "6.4.299","react": "19.3.0",+ "saxes": "6.0.0","semver": "7.8.5","smol-toml": "1.9.1","string-width": "8.3.0",@@ -113,7 +114,6 @@"json5": "2.2.3","minimatch": "10.2.6","prettier": "3.9.10",- "saxes": "6.0.0","typescript": "7.0.2"}}diff --git a/sdk/typescript/pnpm-lock.yaml b/sdk/typescript/pnpm-lock.yamlindex be3d93006..15ba2bc52 100644--- a/sdk/typescript/pnpm-lock.yaml+++ b/sdk/typescript/pnpm-lock.yaml@@ -52,6 +52,9 @@ importers:react:specifier: 19.3.0version: 19.3.0+ saxes:+ specifier: 6.0.0+ version: 6.0.0semver:specifier: 7.8.5version: 7.8.5@@ -116,9 +119,6 @@ importers:prettier:specifier: 3.9.10version: 3.9.10- saxes:- specifier: 6.0.0- version: 6.0.0typescript:specifier: 7.0.2version: 7.0.2diff --git a/sdk/typescript/src/knowledge-base.ts b/sdk/typescript/src/knowledge-base.tsindex 4d924b3a5..8052c87de 100644--- a/sdk/typescript/src/knowledge-base.ts+++ b/sdk/typescript/src/knowledge-base.ts@@ -12,6 +12,7 @@ import {import { tmpdir } from "node:os";import { basename, extname, join, resolve } from "node:path";import { unzipSync } from "fflate";+import { SaxesParser } from "saxes";import { ConfigurationError, errorMessage } from "./errors.js";import { expandHome } from "./runtime.js";import {@@ -303,18 +304,88 @@ function extractDocx(path: string, bytes: Uint8Array): string {const document = files["word/document.xml"];if (document === undefined) throw new Error("Missing word/document.xml.");const xml = decodeText(path, document);- if (- !/<(?:\w+:)?document\b[^>]*>[\s\S]*<\/(?:\w+:)?document\s*>/u.test(xml)- ) {- throw new Error("Malformed word/document.xml.");- }- return decodeXml(- xml- .replace(/<\/(?:\w+:)?p\s*>/gu, "\n")- .replace(/<(?:\w+:)?(?:br|cr)\b[^>]*>/gu, "\n")- .replace(/<(?:\w+:)?tab\b[^>]*>/gu, "\t")- .replace(/<[^>]+>/gu, ""),- );+ let text: string[] = [];+ let fields: {+ instructions: boolean;+ literal: string[];+ result: string[];+ }[] = [];+ const stories: { text: string[]; fields: typeof fields }[] = [];+ const parser = new SaxesParser();+ let documentElement = false;+ let deletedDepth = 0;+ let instructionText = false;+ const append = (value: string) => {+ if (deletedDepth > 0) return;+ const field = fields.at(-1);+ if (field === undefined) text.push(value);+ else if (!field.instructions) field.result.push(value);+ else if (!instructionText) field.literal.push(value);+ };+ const finishField = (closed: boolean) => {+ const field = fields.pop();+ if (field !== undefined) {+ // Unclosed fields retain ordinary runs; completed fields show only results.+ append((closed ? "" : field.literal.join("")) + field.result.join(""));+ }+ };+ parser.on("opentag", ({ name, attributes }) => {+ const localName = name.split(":").at(-1);+ if (+ localName === "del" ||+ localName === "delText" ||+ localName === "delInstrText"+ ) {+ deletedDepth++;+ }+ if (deletedDepth > 0) return;+ if (localName === "txbxContent") {+ // Text boxes have their own field boundaries.+ stories.push({ text, fields });+ text = [];+ fields = [];+ } else if (localName === "document") documentElement = true;+ else if (localName === "instrText") instructionText = true;+ else if (localName === "fldChar") {+ const type = decodeXml(+ Object.entries(attributes).find(+ ([name]) => name.split(":").at(-1) === "fldCharType",+ )?.[1] ?? "",+ );+ if (type === "begin")+ fields.push({ instructions: true, literal: [], result: [] });+ else if (type === "separate" && fields.length > 0)+ fields.at(-1)!.instructions = false;+ else if (type === "end") finishField(true);+ } else if (localName === "br" || localName === "cr") append("\n");+ else if (localName === "tab") append("\t");+ });+ parser.on("closetag", ({ name, isSelfClosing }) => {+ const localName = name.split(":").at(-1);+ if (deletedDepth === 0 && localName === "txbxContent") {+ while (fields.length > 0) finishField(false);+ const content = text.join("");+ ({ text, fields } = stories.pop()!);+ append(content);+ }+ if (localName === "instrText") instructionText = false;+ if (!isSelfClosing && localName === "p") append("\n");+ if (+ localName === "del" ||+ localName === "delText" ||+ localName === "delInstrText"+ ) {+ deletedDepth--;+ }+ });+ parser.on("text", (value) => append(decodeXml(value)));+ parser.on("cdata", (value) => append(value.replaceAll("&", "&")));+ // Keep the existing entity decoder's handling of unusable references.+ // CDATA is literal text, so only undo this escaping in its handler.+ parser.write(xml.replaceAll("&", "&")).close();+ if (!documentElement) throw new Error("Malformed word/document.xml.");+ while (fields.length > 0) finishField(false);+ return text.join("");} catch (error) {throw new Error(`Cannot extract text from knowledge base DOCX: ${path}`, {cause: error,diff --git a/sdk/typescript/tests-ts/knowledge-base.test.ts b/sdk/typescript/tests-ts/knowledge-base.test.tsindex dccf92152..7baee8b19 100644--- a/sdk/typescript/tests-ts/knowledge-base.test.ts+++ b/sdk/typescript/tests-ts/knowledge-base.test.ts@@ -39,9 +39,15 @@ function docx(secondLine?: string,breakElement = "<w:br/>",): Uint8Array {+ return docxBody(+ `<w:p><w:r><w:t>${text}</w:t>${secondLine === undefined ? "" : `${breakElement}<w:t>${secondLine}</w:t>`}</w:r></w:p>`,+ );+}++function docxBody(body: string): Uint8Array {return zipSync({"word/document.xml": strToU8(- `<?xml version="1.0"?><w:document xmlns:w="http://schemas.openxmlformats.org/wordprocessingml/2006/main"><w:body><w:p><w:r><w:t>${text}</w:t>${secondLine === undefined ? "" : `${breakElement}<w:t>${secondLine}</w:t>`}</w:r></w:p></w:body></w:document>`,+ `<?xml version="1.0"?><w:document xmlns:w="http://schemas.openxmlformats.org/wordprocessingml/2006/main"><w:body>${body}</w:body></w:document>`,),});}@@ -490,6 +496,22 @@ describe("scan knowledge bases", () => {join(root, "self-closing-tab.docx"),docx("User", "Permission", "<w:tab/>"),);+ await writeFile(+ join(root, "tracked-change.docx"),+ docxBody(+ '<w:p><w:r><w:t xml:space="preserve">Admin API is </w:t></w:r>' ++ '<w:del w:id="1" w:author="Example Author"><w:r><w:delText>public</w:delText></w:r></w:del>' ++ "<w:r><w:t>internal only</w:t></w:r></w:p>",+ ),+ );+ await writeFile(+ join(root, "field-code.docx"),+ docxBody(+ '<w:p><w:r><w:t xml:space="preserve">See </w:t><w:fldChar w:fldCharType="begin"/></w:r>' ++ '<w:r><w:instrText xml:space="preserve"> HYPERLINK "https://example.com/runbook" </w:instrText></w:r>' ++ '<w:r><w:fldChar w:fldCharType="separate"/><w:t>the runbook</w:t><w:fldChar w:fldCharType="end"/></w:r></w:p>',+ ),+ );const knowledgeBase = await prepareKnowledgeBase([root]);temporaryDirectories.track(knowledgeBase.path);@@ -501,6 +523,133 @@ describe("scan knowledge bases", () => {expect(documents).toContain("Authentication\nReview sessions\n");expect(documents).toContain("Role\tCapability\n");expect(documents).toContain("User\tPermission\n");+ expect(documents).toContain("Admin API is internal only\n");+ expect(documents).toContain("See the runbook\n");+ });++ test.each([+ [+ "an unclosed text-box field inside a main-story field result",+ '<w:p><w:r><w:fldChar w:fldCharType="begin"/><w:instrText>HYPERLINK "https://example.com"</w:instrText>' ++ '<w:fldChar w:fldCharType="separate"/><w:t>Main-before </w:t></w:r>' ++ '<w:r><w:pict xmlns:v="urn:schemas-microsoft-com:vml"><v:rect id="TextBox1" style="width:180pt;height:72pt"><v:textbox><w:txbxContent>' ++ '<w:p><w:r><w:fldChar w:fldCharType="begin"/><w:instrText>AUTHOR</w:instrText><w:t>Text-box guidance.</w:t></w:r></w:p>' ++ "</w:txbxContent></v:textbox></v:rect></w:pict></w:r>" ++ '<w:r><w:fldChar w:fldCharType="end"/><w:instrText>Main-after</w:instrText></w:r></w:p>',+ "Main-before Text-box guidance.\nMain-after\n",+ ],+ [+ "independent fields in sibling text boxes",+ "<w:p><w:r><w:t>Main-before </w:t></w:r>" ++ '<w:r><w:pict xmlns:v="urn:schemas-microsoft-com:vml"><v:rect id="TextBox1" style="width:180pt;height:72pt"><v:textbox><w:txbxContent>' ++ '<w:p><w:r><w:fldChar w:fldCharType="begin"/><w:instrText>AUTHOR</w:instrText><w:t>First text-box guidance.</w:t></w:r></w:p>' ++ "</w:txbxContent></v:textbox></v:rect></w:pict></w:r>" ++ '<w:r><w:pict xmlns:v="urn:schemas-microsoft-com:vml"><v:rect id="TextBox2" style="width:180pt;height:72pt"><v:textbox><w:txbxContent>' ++ '<w:p><w:r><w:t>Second text-box guidance.</w:t><w:fldChar w:fldCharType="end"/></w:r></w:p>' ++ "</w:txbxContent></v:textbox></v:rect></w:pict></w:r>" ++ "<w:r><w:t>Main-after</w:t></w:r></w:p>",+ "Main-before First text-box guidance.\nSecond text-box guidance.\nMain-after\n",+ ],+ [+ "ordinary runs after an unclosed field",+ '<w:p><w:r><w:fldChar w:fldCharType="begin"/><w:instrText>AUTHOR</w:instrText></w:r></w:p>' ++ "<w:p><w:r><w:t>Require authorization</w:t></w:r></w:p>",+ "\nRequire authorization\n",+ ],+ [+ "nested field results inside an unclosed field",+ '<w:p><w:r><w:fldChar w:fldCharType="begin"/><w:instrText>IF </w:instrText>' ++ '<w:fldChar w:fldCharType="begin"/><w:instrText>AUTHOR</w:instrText>' ++ '<w:fldChar w:fldCharType="separate"/><w:t>Example Author</w:t><w:fldChar w:fldCharType="end"/>' ++ "<w:t> requires authorization</w:t></w:r></w:p>",+ "Example Author requires authorization\n",+ ],+ [+ "literal markup in standalone instruction text",+ "<w:p><w:r><w:instrText><![CDATA[Require <authorization> & audit]]></w:instrText></w:r></w:p>",+ "Require <authorization> & audit\n",+ ],+ [+ "field markup in XML comments",+ '<w:p><w:r><w:t>Require<!-- <w:fldChar w:fldCharType="begin"/> --></w:t>' ++ "<w:instrText> authorization</w:instrText></w:r></w:p>",+ "Require authorization\n",+ ],+ [+ "CDATA instructions and character references in field attributes",+ '<w:p><w:r><w:fldChar w:fldCharType="begin"/>' ++ '<w:instrText><![CDATA[HYPERLINK "https://example.com/runbook"]]></w:instrText>' ++ '<w:fldChar w:fldCharType="separate"/><w:t>Read & audit</w:t>' ++ '<w:fldChar w:fldCharType="end"/></w:r></w:p>',+ "Read & audit\n",+ ],+ [+ "standalone instruction text",+ "<w:p><w:r><w:t>Require </w:t><w:instrText>authorization</w:instrText><w:t> before access</w:t></w:r></w:p>",+ "Require authorization before access\n",+ ],+ [+ "instruction text used as a field result",+ '<w:p><w:r><w:fldChar w:fldCharType="begin"/><w:instrText>AUTHOR</w:instrText>' ++ '<w:fldChar w:fldCharType="separate"/><w:instrText>Example Author</w:instrText>' ++ '<w:fldChar w:fldCharType="end"/></w:r></w:p>',+ "Example Author\n",+ ],+ [+ "nested fields within outer instructions",+ '<w:p><w:r><w:fldChar w:fldCharType="begin"/><w:instrText>IF </w:instrText>' ++ '<w:fldChar w:fldCharType="begin"/><w:instrText>AUTHOR</w:instrText>' ++ '<w:fldChar w:fldCharType="separate"/><w:t>Example Author</w:t><w:fldChar w:fldCharType="end"/>' ++ '<w:instrText> = "Example Author" "internal" "public"</w:instrText>' ++ '<w:fldChar w:fldCharType="separate"/><w:t>internal</w:t><w:fldChar w:fldCharType="end"/></w:r></w:p>',+ "internal\n",+ ],+ [+ "nested fields within an outer result",+ '<w:p><w:r><w:fldChar w:fldCharType="begin"/><w:instrText>IF 1 = 1</w:instrText>' ++ '<w:fldChar w:fldCharType="separate"/><w:t>See </w:t>' ++ '<w:fldChar w:fldCharType="begin"/><w:instrText>HYPERLINK "https://example.com/runbook"</w:instrText>' ++ '<w:fldChar w:fldCharType="separate"/><w:t>the runbook</w:t><w:fldChar w:fldCharType="end"/>' ++ '<w:instrText> before access</w:instrText><w:fldChar w:fldCharType="end"/></w:r></w:p>',+ "See the runbook before access\n",+ ],+ [+ "field results spanning paragraphs",+ '<w:p><w:r><w:fldChar w:fldCharType="begin"/><w:instrText>TOC</w:instrText>' ++ '<w:fldChar w:fldCharType="separate"/><w:t>First result paragraph</w:t></w:r></w:p>' ++ '<w:p><w:r><w:instrText>Second result paragraph</w:instrText><w:fldChar w:fldCharType="end"/></w:r></w:p>',+ "First result paragraph\nSecond result paragraph\n",+ ],+ [+ "a field without a result followed by standalone instruction text",+ '<w:p><w:r><w:fldChar w:fldCharType="begin"/><w:instrText>XE "authorization"</w:instrText>' ++ '<w:fldChar w:fldCharType="end"/><w:instrText>Visible scope</w:instrText></w:r></w:p>',+ "Visible scope\n",+ ],+ [+ "deleted field separators and instructions",+ '<w:p><w:r><w:fldChar w:fldCharType="begin"/><w:instrText>IF </w:instrText></w:r>' ++ '<w:del w:id="2" w:author="Example Author"><w:r><w:fldChar w:fldCharType="separate"/>' ++ "<w:delInstrText>old instruction</w:delInstrText></w:r></w:del>" ++ '<w:r><w:instrText>1 = 1 "internal" "public"</w:instrText><w:fldChar w:fldCharType="separate"/>' ++ '<w:t>internal</w:t><w:fldChar w:fldCharType="end"/></w:r></w:p>',+ "internal\n",+ ],+ [+ "deleted field beginnings before standalone instruction text",+ '<w:p><w:del w:id="3" w:author="Example Author"><w:r><w:fldChar w:fldCharType="begin"/>' ++ "<w:delInstrText>removed field</w:delInstrText></w:r></w:del>" ++ "<w:r><w:instrText>Visible scope</w:instrText></w:r></w:p>",+ "Visible scope\n",+ ],+ ])("preserves DOCX text with %s", async (_description, body, expected) => {+ const root = await temporaryDirectory();+ const source = join(root, "field-context.docx");+ await writeFile(source, docxBody(body));++ const snapshot = await readKnowledgeBaseSnapshot([source]);++ expect(Object.values(snapshot.documents)).toEqual([expected]);});test.each([