OpenAI's Codex Security CLI and TypeScript SDK for finding, validating, and fixing security vulnerabilities. npm: https://www.npmjs.com/package/@openai/codex-security
TypeScript ai-securityapplication-securityclicode-scanningcodexcodex-securitycybersecuritydevsecopsnodejsnpmopenaisdksecuritytypescriptvulnerability-scanning
fix(csv): preserve record endings in large finding exports (#1785)
Michael D'Angelocdd8a40a2db9
diff --git a/sdk/typescript/src/findings-import.ts b/sdk/typescript/src/findings-import.tsindex 38140873b..ade7a18fc 100644--- a/sdk/typescript/src/findings-import.ts+++ b/sdk/typescript/src/findings-import.ts@@ -204,6 +204,21 @@ export function bindImportedFindings(});}+function exportedCsvNewline(source: string): "\r\n" | "\r" | "\n" | undefined {+ const start = source.startsWith("\uFEFF") ? 1 : 0;+ for (const columns of [CSV_COLUMNS, REQUIRED_CSV_COLUMNS]) {+ const header = columns.join(",");+ if (!source.startsWith(header, start)) continue;+ const end = start + header.length;+ for (const newline of ["\r\n", "\r", "\n"] as const) {+ // Require the exporter's first cell: skipped empty records can make+ // other mixed-ending CSV valid under Papa's existing auto-detection.+ if (source.startsWith(`${newline}occ_`, end)) return newline;+ }+ }+ return undefined;+}+export function parseFindingsCsv(source: string): CsvFindingRow[] {const {data: rows,@@ -212,6 +227,7 @@ export function parseFindingsCsv(source: string): CsvFindingRow[] {} = Papa.parse<Record<string, string>>(source, {header: true,delimiter: ",",+ newline: exportedCsvNewline(source),skipEmptyLines: "greedy",transform: decodeExportedCsvCell,});diff --git a/sdk/typescript/tests-ts/findings-import-record-endings.test.ts b/sdk/typescript/tests-ts/findings-import-record-endings.test.tsnew file mode 100644index 000000000..708dfdbc9--- /dev/null+++ b/sdk/typescript/tests-ts/findings-import-record-endings.test.ts@@ -0,0 +1,146 @@+import { execFileSync } from "node:child_process";+import { readFile } from "node:fs/promises";+import { join } from "node:path";+import { describe, expect, test } from "bun:test";+import { parseImportedFindings } from "../src/findings-import.js";+import type { FindingsDocument } from "../src/models.js";+import { PLUGIN_ROOT } from "./plugin-root.js";+import { PYTHON } from "./support/security-policy.js";++const HEADER =+ "occurrence_id,finding_id,title,summary,severity,confidence,status," ++ "close_reason,note,remediation,path,start_line,end_line";+const ROW =+ "occ_000000000000000000000001,csf_000000000000000000000001," ++ "Reported issue,Reported summary,high,high,open,,," ++ "Check destinations.,src/extract.ts,41,44";+const ROWS = [+ ROW,+ ROW.replace("occ_000000000000000000000001", "occ_000000000000000000000002"),+];+const OCCURRENCE_IDS = ROWS.map((row) => row.split(",")[0]!);++describe("findings CSV record endings", () => {+ test.each([false, true])(+ "round-trips a large quoted field through the exporter (deep=%j)",+ async (deep) => {+ const document: FindingsDocument = JSON.parse(+ await readFile(+ join(PLUGIN_ROOT, "examples", "completed-scan", "findings.json"),+ "utf8",+ ),+ );+ // The closing quote falls beyond Papa Parse's newline-detection sample.+ const summary = "A\rB\rC\r" + "x".repeat(1024 * 1024);+ const finding = document.findings[0]!;+ document.findings = [+ { ...finding, summary, extensions: { candidateId: "candidate-one" } },+ {+ ...finding,+ occurrenceId: "occ_111111111111111111111111",+ summary: "Following ordinary summary.",+ extensions: { candidateId: "candidate-two" },+ },+ ];+ const csv = execFileSync(+ PYTHON,+ [+ "-c",+ [+ "import json, sys",+ "sys.path.insert(0, sys.argv[1])",+ "from finalize_scan_contract import build_csv_projection",+ "payload = json.load(sys.stdin.buffer)",+ 'sys.stdout.buffer.write(build_csv_projection(payload["document"], payload["coverage"]))',+ ].join("\n"),+ join(PLUGIN_ROOT, "scripts"),+ ],+ {+ cwd: join(PLUGIN_ROOT, "scripts"),+ input: JSON.stringify({+ document,+ coverage: deep ? { mode: "deep_repository" } : {},+ }),+ encoding: "utf8",+ maxBuffer: 2 * 1024 * 1024,+ },+ );+ const header = deep+ ? HEADER.replace("finding_id,", "finding_id,candidate_id,")+ : HEADER;+ expect(csv.startsWith(`${header}\r\n${finding.occurrenceId},`)).toBe(+ true,+ );+ for (const prefix of ["", "\uFEFF"]) {+ const imported = await parseImportedFindings(+ prefix + csv,+ "csv",+ PLUGIN_ROOT,+ );+ expect(imported.map(({ summary }) => summary)).toEqual(+ document.findings.map(({ summary }) => summary),+ );+ expect(+ imported.map(({ findingId, occurrenceId }) => [+ findingId,+ occurrenceId,+ ]),+ ).toEqual(+ document.findings.map(({ findingId, occurrenceId }) => [+ findingId,+ occurrenceId,+ ]),+ );+ if (deep) {+ expect(+ imported.map((item) => item.extensions?.["candidateId"]),+ ).toEqual(["candidate-one", "candidate-two"]);+ }+ }+ },+ );++ test.each(["\r\n", "\n", "\r"])(+ "preserves canonical rows separated by %j",+ async (newline) => {+ const imported = await parseImportedFindings(+ [HEADER, ...ROWS, ""].join(newline),+ "csv",+ PLUGIN_ROOT,+ );+ expect(imported.map((finding) => finding.occurrenceId)).toEqual(+ OCCURRENCE_IDS,+ );+ expect(imported.map((finding) => finding.summary)).toEqual([+ "Reported summary",+ "Reported summary",+ ]);+ },+ );++ test("preserves external layouts and mixed endings with a skipped empty record", async () => {+ const columns = HEADER.split(",");+ const reordered = [1, 0, ...columns.slice(2).map((_, index) => index + 2)];+ const sources = [+ `${columns.map((column) => `"${column}"`).join(",")}\r\n${ROWS.join("\r\n")}\r\n`,+ `${reordered.map((index) => columns[index]).join(",")}\r\n${ROWS.map(+ (row) => {+ const values = row.split(",");+ return reordered.map((index) => values[index]).join(",");+ },+ ).join("\r\n")}\r\n`,+ `${HEADER}\r\n${ROWS.map((row) => row.replace(/^([^,]+)/u, '"$1"')).join("\r\n")}\r\n`,+ `${HEADER}\r\n\r${ROWS.join("\r")}\r`,+ ];+ for (const source of sources) {+ const imported = await parseImportedFindings(source, "csv", PLUGIN_ROOT);+ expect(imported.map((finding) => finding.occurrenceId)).toEqual(+ OCCURRENCE_IDS,+ );+ expect(imported.map((finding) => finding.summary)).toEqual([+ "Reported summary",+ "Reported summary",+ ]);+ }+ });+});