Skip to content

OpenAI's Codex Security CLI and TypeScript SDK for finding, validating, and fixing security vulnerabilities. npm: https://www.npmjs.com/package/@openai/codex-security

fix(knowledge-base): drop deleted revisions and field codes from DOCX (#1714)

12c8a7e3bddc67
diff --git a/sdk/typescript/package.json b/sdk/typescript/package.json
index 5ff1ed45b..4583a17ae 100644
--- a/sdk/typescript/package.json
+++ b/sdk/typescript/package.json
@@ -90,6 +90,7 @@
"papaparse": "5.7.0",
"pdfjs-dist": "6.4.299",
"react": "19.3.0",
+ "saxes": "6.0.0",
"semver": "7.8.5",
"smol-toml": "1.9.1",
"string-width": "8.3.0",
@@ -113,7 +114,6 @@
"json5": "2.2.3",
"minimatch": "10.2.6",
"prettier": "3.9.10",
- "saxes": "6.0.0",
"typescript": "7.0.2"
}
}
diff --git a/sdk/typescript/pnpm-lock.yaml b/sdk/typescript/pnpm-lock.yaml
index be3d93006..15ba2bc52 100644
--- a/sdk/typescript/pnpm-lock.yaml
+++ b/sdk/typescript/pnpm-lock.yaml
@@ -52,6 +52,9 @@ importers:
react:
specifier: 19.3.0
version: 19.3.0
+ saxes:
+ specifier: 6.0.0
+ version: 6.0.0
semver:
specifier: 7.8.5
version: 7.8.5
@@ -116,9 +119,6 @@ importers:
prettier:
specifier: 3.9.10
version: 3.9.10
- saxes:
- specifier: 6.0.0
- version: 6.0.0
typescript:
specifier: 7.0.2
version: 7.0.2
diff --git a/sdk/typescript/src/knowledge-base.ts b/sdk/typescript/src/knowledge-base.ts
index 4d924b3a5..8052c87de 100644
--- a/sdk/typescript/src/knowledge-base.ts
+++ b/sdk/typescript/src/knowledge-base.ts
@@ -12,6 +12,7 @@ import {
import { tmpdir } from "node:os";
import { basename, extname, join, resolve } from "node:path";
import { unzipSync } from "fflate";
+import { SaxesParser } from "saxes";
import { ConfigurationError, errorMessage } from "./errors.js";
import { expandHome } from "./runtime.js";
import {
@@ -303,18 +304,88 @@ function extractDocx(path: string, bytes: Uint8Array): string {
const document = files["word/document.xml"];
if (document === undefined) throw new Error("Missing word/document.xml.");
const xml = decodeText(path, document);
- if (
- !/<(?:\w+:)?document\b[^>]*>[\s\S]*<\/(?:\w+:)?document\s*>/u.test(xml)
- ) {
- throw new Error("Malformed word/document.xml.");
- }
- return decodeXml(
- xml
- .replace(/<\/(?:\w+:)?p\s*>/gu, "\n")
- .replace(/<(?:\w+:)?(?:br|cr)\b[^>]*>/gu, "\n")
- .replace(/<(?:\w+:)?tab\b[^>]*>/gu, "\t")
- .replace(/<[^>]+>/gu, ""),
- );
+ let text: string[] = [];
+ let fields: {
+ instructions: boolean;
+ literal: string[];
+ result: string[];
+ }[] = [];
+ const stories: { text: string[]; fields: typeof fields }[] = [];
+ const parser = new SaxesParser();
+ let documentElement = false;
+ let deletedDepth = 0;
+ let instructionText = false;
+ const append = (value: string) => {
+ if (deletedDepth > 0) return;
+ const field = fields.at(-1);
+ if (field === undefined) text.push(value);
+ else if (!field.instructions) field.result.push(value);
+ else if (!instructionText) field.literal.push(value);
+ };
+ const finishField = (closed: boolean) => {
+ const field = fields.pop();
+ if (field !== undefined) {
+ // Unclosed fields retain ordinary runs; completed fields show only results.
+ append((closed ? "" : field.literal.join("")) + field.result.join(""));
+ }
+ };
+ parser.on("opentag", ({ name, attributes }) => {
+ const localName = name.split(":").at(-1);
+ if (
+ localName === "del" ||
+ localName === "delText" ||
+ localName === "delInstrText"
+ ) {
+ deletedDepth++;
+ }
+ if (deletedDepth > 0) return;
+ if (localName === "txbxContent") {
+ // Text boxes have their own field boundaries.
+ stories.push({ text, fields });
+ text = [];
+ fields = [];
+ } else if (localName === "document") documentElement = true;
+ else if (localName === "instrText") instructionText = true;
+ else if (localName === "fldChar") {
+ const type = decodeXml(
+ Object.entries(attributes).find(
+ ([name]) => name.split(":").at(-1) === "fldCharType",
+ )?.[1] ?? "",
+ );
+ if (type === "begin")
+ fields.push({ instructions: true, literal: [], result: [] });
+ else if (type === "separate" && fields.length > 0)
+ fields.at(-1)!.instructions = false;
+ else if (type === "end") finishField(true);
+ } else if (localName === "br" || localName === "cr") append("\n");
+ else if (localName === "tab") append("\t");
+ });
+ parser.on("closetag", ({ name, isSelfClosing }) => {
+ const localName = name.split(":").at(-1);
+ if (deletedDepth === 0 && localName === "txbxContent") {
+ while (fields.length > 0) finishField(false);
+ const content = text.join("");
+ ({ text, fields } = stories.pop()!);
+ append(content);
+ }
+ if (localName === "instrText") instructionText = false;
+ if (!isSelfClosing && localName === "p") append("\n");
+ if (
+ localName === "del" ||
+ localName === "delText" ||
+ localName === "delInstrText"
+ ) {
+ deletedDepth--;
+ }
+ });
+ parser.on("text", (value) => append(decodeXml(value)));
+ parser.on("cdata", (value) => append(value.replaceAll("&amp;", "&")));
+ // Keep the existing entity decoder's handling of unusable references.
+ // CDATA is literal text, so only undo this escaping in its handler.
+ parser.write(xml.replaceAll("&", "&amp;")).close();
+ if (!documentElement) throw new Error("Malformed word/document.xml.");
+ while (fields.length > 0) finishField(false);
+ return text.join("");
} catch (error) {
throw new Error(`Cannot extract text from knowledge base DOCX: ${path}`, {
cause: error,
diff --git a/sdk/typescript/tests-ts/knowledge-base.test.ts b/sdk/typescript/tests-ts/knowledge-base.test.ts
index dccf92152..7baee8b19 100644
--- a/sdk/typescript/tests-ts/knowledge-base.test.ts
+++ b/sdk/typescript/tests-ts/knowledge-base.test.ts
@@ -39,9 +39,15 @@ function docx(
secondLine?: string,
breakElement = "<w:br/>",
): Uint8Array {
+ return docxBody(
+ `<w:p><w:r><w:t>${text}</w:t>${secondLine === undefined ? "" : `${breakElement}<w:t>${secondLine}</w:t>`}</w:r></w:p>`,
+ );
+}
+
+function docxBody(body: string): Uint8Array {
return zipSync({
"word/document.xml": strToU8(
- `<?xml version="1.0"?><w:document xmlns:w="http://schemas.openxmlformats.org/wordprocessingml/2006/main"><w:body><w:p><w:r><w:t>${text}</w:t>${secondLine === undefined ? "" : `${breakElement}<w:t>${secondLine}</w:t>`}</w:r></w:p></w:body></w:document>`,
+ `<?xml version="1.0"?><w:document xmlns:w="http://schemas.openxmlformats.org/wordprocessingml/2006/main"><w:body>${body}</w:body></w:document>`,
),
});
}
@@ -490,6 +496,22 @@ describe("scan knowledge bases", () => {
join(root, "self-closing-tab.docx"),
docx("User", "Permission", "<w:tab/>"),
);
+ await writeFile(
+ join(root, "tracked-change.docx"),
+ docxBody(
+ '<w:p><w:r><w:t xml:space="preserve">Admin API is </w:t></w:r>' +
+ '<w:del w:id="1" w:author="Example Author"><w:r><w:delText>public</w:delText></w:r></w:del>' +
+ "<w:r><w:t>internal only</w:t></w:r></w:p>",
+ ),
+ );
+ await writeFile(
+ join(root, "field-code.docx"),
+ docxBody(
+ '<w:p><w:r><w:t xml:space="preserve">See </w:t><w:fldChar w:fldCharType="begin"/></w:r>' +
+ '<w:r><w:instrText xml:space="preserve"> HYPERLINK "https://example.com/runbook" </w:instrText></w:r>' +
+ '<w:r><w:fldChar w:fldCharType="separate"/><w:t>the runbook</w:t><w:fldChar w:fldCharType="end"/></w:r></w:p>',
+ ),
+ );
const knowledgeBase = await prepareKnowledgeBase([root]);
temporaryDirectories.track(knowledgeBase.path);
@@ -501,6 +523,133 @@ describe("scan knowledge bases", () => {
expect(documents).toContain("Authentication\nReview sessions\n");
expect(documents).toContain("Role\tCapability\n");
expect(documents).toContain("User\tPermission\n");
+ expect(documents).toContain("Admin API is internal only\n");
+ expect(documents).toContain("See the runbook\n");
+ });
+
+ test.each([
+ [
+ "an unclosed text-box field inside a main-story field result",
+ '<w:p><w:r><w:fldChar w:fldCharType="begin"/><w:instrText>HYPERLINK "https://example.com"</w:instrText>' +
+ '<w:fldChar w:fldCharType="separate"/><w:t>Main-before </w:t></w:r>' +
+ '<w:r><w:pict xmlns:v="urn:schemas-microsoft-com:vml"><v:rect id="TextBox1" style="width:180pt;height:72pt"><v:textbox><w:txbxContent>' +
+ '<w:p><w:r><w:fldChar w:fldCharType="begin"/><w:instrText>AUTHOR</w:instrText><w:t>Text-box guidance.</w:t></w:r></w:p>' +
+ "</w:txbxContent></v:textbox></v:rect></w:pict></w:r>" +
+ '<w:r><w:fldChar w:fldCharType="end"/><w:instrText>Main-after</w:instrText></w:r></w:p>',
+ "Main-before Text-box guidance.\nMain-after\n",
+ ],
+ [
+ "independent fields in sibling text boxes",
+ "<w:p><w:r><w:t>Main-before </w:t></w:r>" +
+ '<w:r><w:pict xmlns:v="urn:schemas-microsoft-com:vml"><v:rect id="TextBox1" style="width:180pt;height:72pt"><v:textbox><w:txbxContent>' +
+ '<w:p><w:r><w:fldChar w:fldCharType="begin"/><w:instrText>AUTHOR</w:instrText><w:t>First text-box guidance.</w:t></w:r></w:p>' +
+ "</w:txbxContent></v:textbox></v:rect></w:pict></w:r>" +
+ '<w:r><w:pict xmlns:v="urn:schemas-microsoft-com:vml"><v:rect id="TextBox2" style="width:180pt;height:72pt"><v:textbox><w:txbxContent>' +
+ '<w:p><w:r><w:t>Second text-box guidance.</w:t><w:fldChar w:fldCharType="end"/></w:r></w:p>' +
+ "</w:txbxContent></v:textbox></v:rect></w:pict></w:r>" +
+ "<w:r><w:t>Main-after</w:t></w:r></w:p>",
+ "Main-before First text-box guidance.\nSecond text-box guidance.\nMain-after\n",
+ ],
+ [
+ "ordinary runs after an unclosed field",
+ '<w:p><w:r><w:fldChar w:fldCharType="begin"/><w:instrText>AUTHOR</w:instrText></w:r></w:p>' +
+ "<w:p><w:r><w:t>Require authorization</w:t></w:r></w:p>",
+ "\nRequire authorization\n",
+ ],
+ [
+ "nested field results inside an unclosed field",
+ '<w:p><w:r><w:fldChar w:fldCharType="begin"/><w:instrText>IF </w:instrText>' +
+ '<w:fldChar w:fldCharType="begin"/><w:instrText>AUTHOR</w:instrText>' +
+ '<w:fldChar w:fldCharType="separate"/><w:t>Example Author</w:t><w:fldChar w:fldCharType="end"/>' +
+ "<w:t> requires authorization</w:t></w:r></w:p>",
+ "Example Author requires authorization\n",
+ ],
+ [
+ "literal markup in standalone instruction text",
+ "<w:p><w:r><w:instrText><![CDATA[Require <authorization> &amp; audit]]></w:instrText></w:r></w:p>",
+ "Require <authorization> &amp; audit\n",
+ ],
+ [
+ "field markup in XML comments",
+ '<w:p><w:r><w:t>Require<!-- <w:fldChar w:fldCharType="begin"/> --></w:t>' +
+ "<w:instrText> authorization</w:instrText></w:r></w:p>",
+ "Require authorization\n",
+ ],
+ [
+ "CDATA instructions and character references in field attributes",
+ '<w:p><w:r><w:fldChar w:fldCharType="beg&#105;n"/>' +
+ '<w:instrText><![CDATA[HYPERLINK "https://example.com/runbook"]]></w:instrText>' +
+ '<w:fldChar w:fldCharType="separate"/><w:t>Read &amp; audit</w:t>' +
+ '<w:fldChar w:fldCharType="end"/></w:r></w:p>',
+ "Read & audit\n",
+ ],
+ [
+ "standalone instruction text",
+ "<w:p><w:r><w:t>Require </w:t><w:instrText>authorization</w:instrText><w:t> before access</w:t></w:r></w:p>",
+ "Require authorization before access\n",
+ ],
+ [
+ "instruction text used as a field result",
+ '<w:p><w:r><w:fldChar w:fldCharType="begin"/><w:instrText>AUTHOR</w:instrText>' +
+ '<w:fldChar w:fldCharType="separate"/><w:instrText>Example Author</w:instrText>' +
+ '<w:fldChar w:fldCharType="end"/></w:r></w:p>',
+ "Example Author\n",
+ ],
+ [
+ "nested fields within outer instructions",
+ '<w:p><w:r><w:fldChar w:fldCharType="begin"/><w:instrText>IF </w:instrText>' +
+ '<w:fldChar w:fldCharType="begin"/><w:instrText>AUTHOR</w:instrText>' +
+ '<w:fldChar w:fldCharType="separate"/><w:t>Example Author</w:t><w:fldChar w:fldCharType="end"/>' +
+ '<w:instrText> = "Example Author" "internal" "public"</w:instrText>' +
+ '<w:fldChar w:fldCharType="separate"/><w:t>internal</w:t><w:fldChar w:fldCharType="end"/></w:r></w:p>',
+ "internal\n",
+ ],
+ [
+ "nested fields within an outer result",
+ '<w:p><w:r><w:fldChar w:fldCharType="begin"/><w:instrText>IF 1 = 1</w:instrText>' +
+ '<w:fldChar w:fldCharType="separate"/><w:t>See </w:t>' +
+ '<w:fldChar w:fldCharType="begin"/><w:instrText>HYPERLINK "https://example.com/runbook"</w:instrText>' +
+ '<w:fldChar w:fldCharType="separate"/><w:t>the runbook</w:t><w:fldChar w:fldCharType="end"/>' +
+ '<w:instrText> before access</w:instrText><w:fldChar w:fldCharType="end"/></w:r></w:p>',
+ "See the runbook before access\n",
+ ],
+ [
+ "field results spanning paragraphs",
+ '<w:p><w:r><w:fldChar w:fldCharType="begin"/><w:instrText>TOC</w:instrText>' +
+ '<w:fldChar w:fldCharType="separate"/><w:t>First result paragraph</w:t></w:r></w:p>' +
+ '<w:p><w:r><w:instrText>Second result paragraph</w:instrText><w:fldChar w:fldCharType="end"/></w:r></w:p>',
+ "First result paragraph\nSecond result paragraph\n",
+ ],
+ [
+ "a field without a result followed by standalone instruction text",
+ '<w:p><w:r><w:fldChar w:fldCharType="begin"/><w:instrText>XE "authorization"</w:instrText>' +
+ '<w:fldChar w:fldCharType="end"/><w:instrText>Visible scope</w:instrText></w:r></w:p>',
+ "Visible scope\n",
+ ],
+ [
+ "deleted field separators and instructions",
+ '<w:p><w:r><w:fldChar w:fldCharType="begin"/><w:instrText>IF </w:instrText></w:r>' +
+ '<w:del w:id="2" w:author="Example Author"><w:r><w:fldChar w:fldCharType="separate"/>' +
+ "<w:delInstrText>old instruction</w:delInstrText></w:r></w:del>" +
+ '<w:r><w:instrText>1 = 1 "internal" "public"</w:instrText><w:fldChar w:fldCharType="separate"/>' +
+ '<w:t>internal</w:t><w:fldChar w:fldCharType="end"/></w:r></w:p>',
+ "internal\n",
+ ],
+ [
+ "deleted field beginnings before standalone instruction text",
+ '<w:p><w:del w:id="3" w:author="Example Author"><w:r><w:fldChar w:fldCharType="begin"/>' +
+ "<w:delInstrText>removed field</w:delInstrText></w:r></w:del>" +
+ "<w:r><w:instrText>Visible scope</w:instrText></w:r></w:p>",
+ "Visible scope\n",
+ ],
+ ])("preserves DOCX text with %s", async (_description, body, expected) => {
+ const root = await temporaryDirectory();
+ const source = join(root, "field-context.docx");
+ await writeFile(source, docxBody(body));
+
+ const snapshot = await readKnowledgeBaseSnapshot([source]);
+
+ expect(Object.values(snapshot.documents)).toEqual([expected]);
});
test.each([