Skip to content

OpenAI's Codex Security CLI and TypeScript SDK for finding, validating, and fixing security vulnerabilities. npm: https://www.npmjs.com/package/@openai/codex-security

fix(knowledge-base): preserve PDF line breaks during text extraction (#2196)

Lanre Shittu14adc6038639
diff --git a/sdk/typescript/src/knowledge-base.ts b/sdk/typescript/src/knowledge-base.ts
index 2f9e93536..3782beba1 100644
--- a/sdk/typescript/src/knowledge-base.ts
+++ b/sdk/typescript/src/knowledge-base.ts
@@ -281,11 +281,17 @@ async function extractPdf(
signal?.throwIfAborted();
const content = await page.getTextContent();
signal?.throwIfAborted();
- pages.push(
- content.items
- .map((item) => ("str" in item ? item.str : ""))
- .join(" "),
- );
+ const parts: string[] = [];
+ for (const item of content.items) {
+ if (!("str" in item)) continue;
+ // Separate runs when the PDF supplies no whitespace, including across forms.
+ if (/^\S/u.test(item.str) && /\S$/u.test(parts.at(-1) ?? "")) {
+ parts.push(" ");
+ }
+ if (item.str) parts.push(item.str);
+ if (item.hasEOL) parts.push("\n");
+ }
+ pages.push(parts.join(""));
}
return pages.join("\n");
} finally {
diff --git a/sdk/typescript/tests-ts/knowledge-base.test.ts b/sdk/typescript/tests-ts/knowledge-base.test.ts
index 8cdb8f63a..f1ef8cb3d 100644
--- a/sdk/typescript/tests-ts/knowledge-base.test.ts
+++ b/sdk/typescript/tests-ts/knowledge-base.test.ts
@@ -53,15 +53,25 @@ function docxBody(body: string): Uint8Array {
}
function pdf(text: string): Uint8Array {
- const escaped = text.replace(/[\\()]/gu, "\\$&");
- const stream = `BT /F1 12 Tf 72 720 Td (${escaped}) Tj ET`;
+ return pdfContent(`(${text.replace(/[\\()]/gu, "\\$&")}) Tj`);
+}
+
+function pdfContent(operators: string, form?: string): Uint8Array {
+ const stream = `BT /F1 12 Tf 72 720 Td ${operators} ET`;
+ const font = "/Font << /F1 4 0 R >>";
+ const forms = form ? " /XObject << /Word 6 0 R >>" : "";
const objects = [
"<< /Type /Catalog /Pages 2 0 R >>",
"<< /Type /Pages /Kids [3 0 R] /Count 1 >>",
- "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Resources << /Font << /F1 4 0 R >> >> /Contents 5 0 R >>",
+ `<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Resources << ${font}${forms} >> /Contents 5 0 R >>`,
"<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>",
`<< /Length ${Buffer.byteLength(stream)} >>\nstream\n${stream}\nendstream`,
];
+ if (form) {
+ objects.push(
+ `<< /Type /XObject /Subtype /Form /BBox [0 0 300 100] /Resources << ${font} >> /Length ${Buffer.byteLength(form)} >>\nstream\n${form}\nendstream`,
+ );
+ }
let output = "%PDF-1.4\n";
const offsets = [0];
for (const [index, object] of objects.entries()) {
@@ -474,7 +484,10 @@ describe("scan knowledge bases", () => {
const root = await temporaryDirectory();
await writeFile(
join(root, "architecture.pdf"),
- pdf("Payment service boundary"),
+ pdfContent(
+ "(Policy ) Tj /F1 14 Tf (guide) Tj /F1 12 Tf 0 -14 Td (Require) Tj ET q 1 0 0 1 117.348 706 cm /Word Do Q BT",
+ "BT /F1 12 Tf (authentication) Tj ET",
+ ),
);
await writeFile(
join(root, "threat-model.docx"),
@@ -517,7 +530,7 @@ describe("scan knowledge bases", () => {
temporaryDirectories.track(knowledgeBase.path);
const documents = await extractedDocuments(knowledgeBase.path);
- expect(documents).toContain("Payment service boundary");
+ expect(documents).toContain("Policy guide\nRequire authentication");
expect(documents).toContain("SSRF & IDOR\nReview authentication\n");
expect(documents).toContain("Authorization\nReview permissions\n");
expect(documents).toContain("Authentication\nReview sessions\n");