OpenAI's Codex Security CLI and TypeScript SDK for finding, validating, and fixing security vulnerabilities. npm: https://www.npmjs.com/package/@openai/codex-security
TypeScript ai-securityapplication-securityclicode-scanningcodexcodex-securitycybersecuritydevsecopsnodejsnpmopenaisdksecuritytypescriptvulnerability-scanning
fix(knowledge-base): preserve PDF line breaks during text extraction (#2196)
Lanre Shittu14adc6038639
diff --git a/sdk/typescript/src/knowledge-base.ts b/sdk/typescript/src/knowledge-base.tsindex 2f9e93536..3782beba1 100644--- a/sdk/typescript/src/knowledge-base.ts+++ b/sdk/typescript/src/knowledge-base.ts@@ -281,11 +281,17 @@ async function extractPdf(signal?.throwIfAborted();const content = await page.getTextContent();signal?.throwIfAborted();- pages.push(- content.items- .map((item) => ("str" in item ? item.str : ""))- .join(" "),- );+ const parts: string[] = [];+ for (const item of content.items) {+ if (!("str" in item)) continue;+ // Separate runs when the PDF supplies no whitespace, including across forms.+ if (/^\S/u.test(item.str) && /\S$/u.test(parts.at(-1) ?? "")) {+ parts.push(" ");+ }+ if (item.str) parts.push(item.str);+ if (item.hasEOL) parts.push("\n");+ }+ pages.push(parts.join(""));}return pages.join("\n");} finally {diff --git a/sdk/typescript/tests-ts/knowledge-base.test.ts b/sdk/typescript/tests-ts/knowledge-base.test.tsindex 8cdb8f63a..f1ef8cb3d 100644--- a/sdk/typescript/tests-ts/knowledge-base.test.ts+++ b/sdk/typescript/tests-ts/knowledge-base.test.ts@@ -53,15 +53,25 @@ function docxBody(body: string): Uint8Array {}function pdf(text: string): Uint8Array {- const escaped = text.replace(/[\\()]/gu, "\\$&");- const stream = `BT /F1 12 Tf 72 720 Td (${escaped}) Tj ET`;+ return pdfContent(`(${text.replace(/[\\()]/gu, "\\$&")}) Tj`);+}++function pdfContent(operators: string, form?: string): Uint8Array {+ const stream = `BT /F1 12 Tf 72 720 Td ${operators} ET`;+ const font = "/Font << /F1 4 0 R >>";+ const forms = form ? " /XObject << /Word 6 0 R >>" : "";const objects = ["<< /Type /Catalog /Pages 2 0 R >>","<< /Type /Pages /Kids [3 0 R] /Count 1 >>",- "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Resources << /Font << /F1 4 0 R >> >> /Contents 5 0 R >>",+ `<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Resources << ${font}${forms} >> /Contents 5 0 R >>`,"<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>",`<< /Length ${Buffer.byteLength(stream)} >>\nstream\n${stream}\nendstream`,];+ if (form) {+ objects.push(+ `<< /Type /XObject /Subtype /Form /BBox [0 0 300 100] /Resources << ${font} >> /Length ${Buffer.byteLength(form)} >>\nstream\n${form}\nendstream`,+ );+ }let output = "%PDF-1.4\n";const offsets = [0];for (const [index, object] of objects.entries()) {@@ -474,7 +484,10 @@ describe("scan knowledge bases", () => {const root = await temporaryDirectory();await writeFile(join(root, "architecture.pdf"),- pdf("Payment service boundary"),+ pdfContent(+ "(Policy ) Tj /F1 14 Tf (guide) Tj /F1 12 Tf 0 -14 Td (Require) Tj ET q 1 0 0 1 117.348 706 cm /Word Do Q BT",+ "BT /F1 12 Tf (authentication) Tj ET",+ ),);await writeFile(join(root, "threat-model.docx"),@@ -517,7 +530,7 @@ describe("scan knowledge bases", () => {temporaryDirectories.track(knowledgeBase.path);const documents = await extractedDocuments(knowledgeBase.path);- expect(documents).toContain("Payment service boundary");+ expect(documents).toContain("Policy guide\nRequire authentication");expect(documents).toContain("SSRF & IDOR\nReview authentication\n");expect(documents).toContain("Authorization\nReview permissions\n");expect(documents).toContain("Authentication\nReview sessions\n");