OpenAI's Codex Security CLI and TypeScript SDK for finding, validating, and fixing security vulnerabilities. npm: https://www.npmjs.com/package/@openai/codex-security
TypeScript ai-securityapplication-securityclicode-scanningcodexcodex-securitycybersecuritydevsecopsnodejsnpmopenaisdksecuritytypescriptvulnerability-scanning
fix(bulk-scan): preserve interrupted receipt bytes (#1748)
Michael D'Angelo6d04d5a694d8
diff --git a/sdk/typescript/src/multiscan.ts b/sdk/typescript/src/multiscan.tsindex 5bc52da0c..1d88d7af8 100644--- a/sdk/typescript/src/multiscan.ts+++ b/sdk/typescript/src/multiscan.ts@@ -999,27 +999,25 @@ async function readReceipts(path: string,preserveInterrupted = false,): Promise<Map<string, MultiscanHistory>> {- let contents: string;+ let contents: Buffer;try {- contents = await readFile(path, "utf8");+ contents = await readFile(path);} catch (error) {if ((error as NodeJS.ErrnoException).code === "ENOENT") return new Map();throw error;}- const lines = contents.split("\n");- if (!contents.endsWith("\n")) {- const partial = lines.pop()!;+ const end = contents.lastIndexOf(10) + 1;+ if (contents.at(-1) !== 10) {+ const partial = contents.subarray(end);if (preserveInterrupted) {await writeFile(`${path}.interrupted-${randomUUID()}`, partial, {flag: "wx",mode: 0o600,});}- await truncate(- path,- Buffer.byteLength(contents) - Buffer.byteLength(partial),- );+ await truncate(path, end);}+ const lines = contents.subarray(0, end).toString("utf8").split("\n");const receipts = new Map<string, MultiscanHistory>();for (const line of lines.filter(Boolean)) {const saved = JSON.parse(line);diff --git a/sdk/typescript/tests-ts/multiscan.test.ts b/sdk/typescript/tests-ts/multiscan.test.tsindex 37ba5aa89..50abfdd47 100644--- a/sdk/typescript/tests-ts/multiscan.test.ts+++ b/sdk/typescript/tests-ts/multiscan.test.ts@@ -680,6 +680,181 @@ describe("multiscan", () => {]);});+ const interruptedUtf8Cases: [string, number, boolean][] = [+ ["é", 1, false],+ ["中", 1, false],+ ["中", 2, false],+ ["🧪", 1, false],+ ["🧪", 2, false],+ ["🧪", 3, false],+ ["malformed prefix", 0, true],+ ];+ test.each(interruptedUtf8Cases)(+ "recovery preserves interrupted %s after %d UTF-8 bytes (malformed prefix=%p)",+ async (character, length, malformedPrefix) => {+ const paths = await fixture();+ const task = {+ id: "complete",+ repository: join(paths.root, "unused-source"),+ revision: "a".repeat(40),+ mode: "standard",+ };+ const outputDir = join(paths.output, "artifacts", task.id, "attempt-1");+ await mkdir(outputDir, { recursive: true, mode: 0o700 });+ const artifacts = [+ "scan-manifest.json",+ "findings.json",+ "coverage.json",+ "report.md",+ ].map((name) => join(outputDir, name));+ await Promise.all(artifacts.map((path) => writeFile(path, "{}\n")));+ const before = await Promise.all(artifacts.map((path) => readFile(path)));+ await writeFile(+ paths.input,+ `id,repository,revision\n${task.id},${task.repository},${task.revision}\n`,+ );+ await writeFile(+ join(paths.output, "manifest.json"),+ JSON.stringify({ version: 2, tasks: [task] }, null, 2) + "\n",+ );+ let prefix = Buffer.from(+ JSON.stringify({+ ...task,+ status: "completed",+ attempt: 1,+ outputDir,+ error: "prefix marker é中🧪",+ }) + "\n",+ );+ if (malformedPrefix) {+ const offset = prefix.indexOf("prefix marker");+ expect(offset).toBeGreaterThan(-1);+ prefix = Buffer.concat([+ prefix.subarray(0, offset),+ Buffer.from([0xff, 0xfe]),+ prefix.subarray(offset + "prefix marker".length),+ ]);+ }+ const receipt = Buffer.from(+ JSON.stringify({+ ...task,+ status: "failed",+ attempt: 2,+ error: character,+ }) + "\n",+ );+ // The malformed prefix made the old offset retain this byte and add NULs.+ const tail = malformedPrefix+ ? Buffer.from([0xf0])+ : receipt.subarray(+ 0,+ receipt.lastIndexOf(Buffer.from(character)) + length,+ );+ const ledger = join(paths.output, "results.jsonl");+ await writeFile(ledger, Buffer.concat([prefix, tail]));+ const createSecurity = mock(+ throwing("Completed task must not start a client"),+ );+ const recoverScan = mock(rejecting("Completed task must not be resumed"));+ for (const _ of [1, 2]) {+ expect(+ await runMultiscan({+ inputPath: paths.input,+ outputDir: paths.output,+ mode: "standard",+ workers: 1,+ maxAttempts: 1,+ config: {},+ createSecurity,+ recoverScan,+ }),+ ).toEqual({+ total: 1,+ completed: 1,+ failed: 0,+ skipped: 1,+ resultsPath: ledger,+ });+ expect(createSecurity).not.toHaveBeenCalled();+ expect(recoverScan).not.toHaveBeenCalled();+ expect(await readFile(ledger)).toEqual(prefix);+ const backups = (await readdir(paths.output)).filter((name) =>+ name.startsWith("results.jsonl.interrupted-"),+ );+ expect(backups).toHaveLength(1);+ expect(await readFile(join(paths.output, backups[0]!))).toEqual(tail);+ expect(+ await Promise.all(artifacts.map((path) => readFile(path))),+ ).toEqual(before);+ }+ },+ );++ test.each(["backup", "truncate"] as const)(+ "recovery preserves interrupted receipt bytes when %s fails",+ async (operation) => {+ const { paths } = await repositoryFixture("interrupted-receipt");+ const run = mock(completeRun);+ const configured = options(paths, client(run));+ const initial = await runMultiscan(configured);+ const original = await readFile(initial.resultsPath);+ const tail = Buffer.concat([+ Buffer.from('{"error":"'),+ Buffer.from([0xc3]),+ ]);+ const damaged = Buffer.concat([original, tail]);+ await writeFile(initial.resultsPath, damaged);+ const failure = new Error(`Synthetic receipt ${operation} failure`);+ const recoverScan = mock(async () => undefined);+ const originalWriteFile = filesystem.writeFile;+ const originalTruncate = filesystem.truncate;+ const backupSpy = spyOn(filesystem, "writeFile").mockImplementation(+ async (...args) => {+ if (+ operation === "backup" &&+ String(args[0]).startsWith(`${initial.resultsPath}.interrupted-`)+ )+ throw failure;+ return await originalWriteFile(...args);+ },+ );+ const truncateSpy = spyOn(filesystem, "truncate").mockImplementation(+ async (path, length) => {+ if (path === initial.resultsPath && operation === "truncate")+ throw failure;+ return await originalTruncate(path, length);+ },+ );+ try {+ await expect(runMultiscan({ ...configured, recoverScan })).rejects.toBe(+ failure,+ );+ expect(await readFile(initial.resultsPath)).toEqual(damaged);+ expect(run).toHaveBeenCalledTimes(1);+ expect(recoverScan).not.toHaveBeenCalled();+ const names = await readdir(paths.output);+ expect(names).not.toContain(".lock");+ const backups = names.filter((name) =>+ name.startsWith("results.jsonl.interrupted-"),+ );+ if (operation === "backup") {+ expect(backups).toHaveLength(0);+ expect(truncateSpy).not.toHaveBeenCalled();+ } else {+ expect(backups).toHaveLength(1);+ expect(await readFile(join(paths.output, backups[0]!))).toEqual(tail);+ expect(truncateSpy).toHaveBeenCalledWith(+ initial.resultsPath,+ original.length,+ );+ }+ } finally {+ backupSpy.mockRestore();+ truncateSpy.mockRestore();+ }+ },+ );+test("recovery skips untouched rows and saves an interrupted ledger tail before appending", async () => {const paths = await fixture();const source = await repository(paths.root, "tail");