OpenAI's Codex Security CLI and TypeScript SDK for finding, validating, and fixing security vulnerabilities. npm: https://www.npmjs.com/package/@openai/codex-security
TypeScript ai-securityapplication-securityclicode-scanningcodexcodex-securitycybersecuritydevsecopsnodejsnpmopenaisdksecuritytypescriptvulnerability-scanning
fix: make multiscan supervisor locks safe across container restarts (#287)
Kyle Brownb8875d07970b
diff --git a/sdk/typescript/src/multiscan.ts b/sdk/typescript/src/multiscan.tsindex 4795bcfdb..eaed6f288 100644--- a/sdk/typescript/src/multiscan.ts+++ b/sdk/typescript/src/multiscan.ts@@ -9,8 +9,10 @@ import {rename,rm,truncate,+ utimes,writeFile,} from "node:fs/promises";+import { hostname } from "node:os";import { dirname, isAbsolute, join, relative, resolve, sep } from "node:path";import { promisify } from "node:util";import Papa from "papaparse";@@ -28,6 +30,8 @@ const REQUIRED_ARTIFACTS = ["coverage.json","report.md",];+const LOCK_LEASE_MS = 30_000;+const LOCK_HEARTBEAT_MS = 5_000;interface MultiscanTask {id: string;@@ -276,25 +280,153 @@ async function acquireLock(output: string): Promise<() => Promise<void>> {await mkdir(path, { mode: 0o700 });} catch (error) {if ((error as NodeJS.ErrnoException).code !== "EEXIST") throw error;- const { pid } = JSON.parse(await readFile(ownerPath, "utf8")) as {- pid: number;+ const existing = await inspectLock(path);+ if (!existing.stale) {+ throw new Error("A multiscan supervisor is already running.");+ }+ await recoverLock(output, path, existing.owner);+ return await acquireLock(output);+ }+ const owner = `${JSON.stringify({+ pid: process.pid,+ ownerId: randomUUID(),+ hostname: hostname(),+ processStartedAt: performance.timeOrigin,+ })}\n`;+ await writeFile(ownerPath, owner, { flag: "wx", mode: 0o600 });++ let heartbeat = Promise.resolve();+ const timer = setInterval(() => {+ heartbeat = heartbeat+ .then(async () => {+ if ((await readFile(ownerPath, "utf8")) !== owner) return;+ const now = new Date();+ await utimes(ownerPath, now, now);+ })+ .catch(() => {});+ }, LOCK_HEARTBEAT_MS);+ timer.unref();++ return async () => {+ clearInterval(timer);+ await heartbeat;+ const current = await readFile(ownerPath, "utf8").catch(+ (error: NodeJS.ErrnoException) => {+ if (error.code !== "ENOENT") throw error;+ return undefined;+ },+ );+ if (current === owner) await rm(path, { recursive: true });+ };+}++async function inspectLock(+ path: string,+): Promise<{ owner: string | undefined; stale: boolean }> {+ const ownerPath = join(path, "owner.json");+ let owner: string;+ let modifiedAt: number;+ try {+ owner = await readFile(ownerPath, "utf8");+ modifiedAt = (await lstat(ownerPath)).mtimeMs;+ } catch (error) {+ if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error;+ return {+ owner: undefined,+ stale: Date.now() - (await lstat(path)).mtimeMs > LOCK_LEASE_MS,};- try {- process.kill(pid, 0);+ }++ let identity: {+ pid?: number;+ ownerId?: string;+ hostname?: string;+ processStartedAt?: number;+ };+ try {+ identity = JSON.parse(owner) as typeof identity;+ } catch {+ return { owner, stale: Date.now() - modifiedAt > LOCK_LEASE_MS };+ }++ if (+ typeof identity.ownerId === "string" &&+ typeof identity.hostname === "string" &&+ typeof identity.processStartedAt === "number"+ ) {+ const sameProcess =+ identity.pid === process.pid &&+ identity.hostname === hostname() &&+ identity.processStartedAt === performance.timeOrigin;+ return {+ owner,+ stale: !sameProcess && Date.now() - modifiedAt > LOCK_LEASE_MS,+ };+ }++ if (+ identity.pid === undefined ||+ !Number.isSafeInteger(identity.pid) ||+ identity.pid < 1+ ) {+ return { owner, stale: Date.now() - modifiedAt > LOCK_LEASE_MS };+ }+ try {+ process.kill(identity.pid, 0);+ } catch (error) {+ if ((error as NodeJS.ErrnoException).code === "ESRCH") {+ return { owner, stale: true };+ }+ if ((error as NodeJS.ErrnoException).code === "EPERM") {+ return { owner, stale: false };+ }+ throw error;+ }+ return {+ owner,+ stale:+ identity.pid === process.pid &&+ modifiedAt + 1_000 < performance.timeOrigin,+ };+}++async function recoverLock(+ output: string,+ path: string,+ expectedOwner: string | undefined,+): Promise<void> {+ const recoveryPath = join(path, ".recovering");+ let claim;+ try {+ claim = await open(recoveryPath, "wx", 0o600);+ } catch (error) {+ if ((error as NodeJS.ErrnoException).code === "EEXIST") {+ if (Date.now() - (await lstat(recoveryPath)).mtimeMs > LOCK_LEASE_MS) {+ await rm(recoveryPath, { force: true });+ return await recoverLock(output, path, expectedOwner);+ }+ throw new Error("A multiscan supervisor is already running.");+ }+ throw error;+ }+ await claim.close();++ let moved = false;+ try {+ const current = await inspectLock(path);+ if (+ current.owner !== expectedOwner ||+ (expectedOwner !== undefined && !current.stale)+ ) {throw new Error("A multiscan supervisor is already running.");- } catch (failure) {- if ((failure as NodeJS.ErrnoException).code !== "ESRCH") throw failure;}const stale = join(output, `.lock.stale-${randomUUID()}`);await rename(path, stale);+ moved = true;await rm(stale, { recursive: true });- return await acquireLock(output);+ } finally {+ if (!moved) await rm(recoveryPath, { force: true });}- await writeFile(ownerPath, `${JSON.stringify({ pid: process.pid })}\n`, {- flag: "wx",- mode: 0o600,- });- return async () => rm(path, { recursive: true });}async function ensureManifest(diff --git a/sdk/typescript/tests-ts/multiscan.test.ts b/sdk/typescript/tests-ts/multiscan.test.tsindex 429bb793e..3d803096f 100644--- a/sdk/typescript/tests-ts/multiscan.test.ts+++ b/sdk/typescript/tests-ts/multiscan.test.ts@@ -2,17 +2,21 @@ import { execFileSync } from "node:child_process";import {access,appendFile,+ lstat,mkdir,mkdtemp,readFile,readdir,+ rename,rm,symlink,+ utimes,writeFile,} from "node:fs/promises";-import { tmpdir } from "node:os";+import * as filesystem from "node:fs/promises";+import { hostname, tmpdir } from "node:os";import { join } from "node:path";-import { afterEach, describe, expect, test } from "bun:test";+import { afterEach, describe, expect, spyOn, test } from "bun:test";import type { ScanResult } from "../src/result.js";import { buildGitHubCredentialArgs, runMultiscan } from "../src/multiscan.js";import { resolveTrustedExecutable } from "../src/trusted-executable.js";@@ -421,6 +425,18 @@ describe("multiscan", () => {const first = runMultiscan(options(paths, security));await running;try {+ const lock = join(paths.output, ".lock");+ const ownerPath = join(lock, "owner.json");+ expect(JSON.parse(await readFile(ownerPath, "utf8"))).toMatchObject({+ pid: process.pid,+ ownerId: expect.any(String),+ hostname: hostname(),+ processStartedAt: expect.any(Number),+ });+ if (process.platform !== "win32") {+ expect((await lstat(lock)).mode & 0o777).toBe(0o700);+ expect((await lstat(ownerPath)).mode & 0o777).toBe(0o600);+ }await expect(runMultiscan(options(paths, security))).rejects.toThrow(/running|locked|supervisor/iu,);@@ -446,6 +462,368 @@ describe("multiscan", () => {await expect(access(lock)).rejects.toThrow();});+ test("recovers a legacy supervisor lock when this live PID was reused", async () => {+ const paths = await fixture();+ const source = await repository(paths.root, "legacy-pid-reuse");+ await writeFile(+ paths.input,+ `id,repository,revision\nlegacy,${source.path},${source.revision}\n`,+ );+ const lock = join(paths.output, ".lock");+ const ownerPath = join(lock, "owner.json");+ await mkdir(lock, { recursive: true, mode: 0o700 });+ await writeFile(ownerPath, JSON.stringify({ pid: process.pid }), {+ mode: 0o600,+ });+ const beforeProcessStarted = new Date(performance.timeOrigin - 60_000);+ await utimes(ownerPath, beforeProcessStarted, beforeProcessStarted);++ const summary = await runMultiscan(+ options(+ paths,+ client(async (_repository, scanOptions = {}) =>+ completedScan(scanOptions.outputDir!),+ ),+ ),+ );++ expect(summary).toMatchObject({ completed: 1, failed: 0 });+ await expect(access(lock)).rejects.toThrow();+ });++ test("preserves an active legacy supervisor lock", async () => {+ const paths = await fixture();+ const source = await repository(paths.root, "legacy-owner");+ await writeFile(+ paths.input,+ `id,repository,revision\nlegacy,${source.path},${source.revision}\n`,+ );+ const lock = join(paths.output, ".lock");+ const ownerPath = join(lock, "owner.json");+ await mkdir(lock, { recursive: true, mode: 0o700 });+ await writeFile(ownerPath, JSON.stringify({ pid: process.pid }), {+ mode: 0o600,+ });++ await expect(+ runMultiscan(+ options(+ paths,+ client(async (_repository, scanOptions = {}) =>+ completedScan(scanOptions.outputDir!),+ ),+ ),+ ),+ ).rejects.toThrow("A multiscan supervisor is already running.");+ expect(JSON.parse(await readFile(ownerPath, "utf8"))).toEqual({+ pid: process.pid,+ });+ });++ for (const previousHostname of [hostname(), "previous-container"]) {+ test(`recovers an expired supervisor lease from ${previousHostname === hostname() ? "a reused live PID" : "a replacement container"}`, async () => {+ const paths = await fixture();+ const source = await repository(paths.root, "expired-supervisor");+ await writeFile(+ paths.input,+ `id,repository,revision\nexpired,${source.path},${source.revision}\n`,+ );+ const lock = join(paths.output, ".lock");+ const ownerPath = join(lock, "owner.json");+ await mkdir(lock, { recursive: true, mode: 0o700 });+ await writeFile(+ ownerPath,+ JSON.stringify({+ pid: process.pid,+ ownerId: "previous-supervisor",+ hostname: previousHostname,+ processStartedAt: performance.timeOrigin - 60_000,+ }),+ { mode: 0o600 },+ );+ const expired = new Date(Date.now() - 120_000);+ await utimes(ownerPath, expired, expired);++ const summary = await runMultiscan(+ options(+ paths,+ client(async (_repository, scanOptions = {}) =>+ completedScan(scanOptions.outputDir!),+ ),+ ),+ );++ expect(summary).toMatchObject({ completed: 1, failed: 0 });+ await expect(access(lock)).rejects.toThrow();+ });+ }++ test("does not reclaim a live supervisor in another container", async () => {+ const paths = await fixture();+ const source = await repository(paths.root, "remote-supervisor");+ await writeFile(+ paths.input,+ `id,repository,revision\nremote,${source.path},${source.revision}\n`,+ );+ const lock = join(paths.output, ".lock");+ const ownerPath = join(lock, "owner.json");+ await mkdir(lock, { recursive: true, mode: 0o700 });+ await writeFile(+ ownerPath,+ JSON.stringify({+ pid: 999_999_999,+ ownerId: "live-remote-supervisor",+ hostname: "another-container",+ processStartedAt: performance.timeOrigin,+ }),+ { mode: 0o600 },+ );++ await expect(+ runMultiscan(+ options(+ paths,+ client(async (_repository, scanOptions = {}) =>+ completedScan(scanOptions.outputDir!),+ ),+ ),+ ),+ ).rejects.toThrow("A multiscan supervisor is already running.");+ expect(JSON.parse(await readFile(ownerPath, "utf8"))).toMatchObject({+ ownerId: "live-remote-supervisor",+ });+ });++ test("recovers interrupted lock creation without an owner record", async () => {+ const paths = await fixture();+ const source = await repository(paths.root, "interrupted-owner");+ await writeFile(+ paths.input,+ `id,repository,revision\ninterrupted,${source.path},${source.revision}\n`,+ );+ const lock = join(paths.output, ".lock");+ await mkdir(lock, { recursive: true, mode: 0o700 });+ const expired = new Date(Date.now() - 120_000);+ await utimes(lock, expired, expired);++ const summary = await runMultiscan(+ options(+ paths,+ client(async (_repository, scanOptions = {}) =>+ completedScan(scanOptions.outputDir!),+ ),+ ),+ );++ expect(summary).toMatchObject({ completed: 1, failed: 0 });+ await expect(access(lock)).rejects.toThrow();+ });++ test("preserves a supervisor lock while its owner record is being created", async () => {+ const paths = await fixture();+ const source = await repository(paths.root, "initializing-owner");+ await writeFile(+ paths.input,+ `id,repository,revision\ninitializing,${source.path},${source.revision}\n`,+ );+ const lock = join(paths.output, ".lock");+ await mkdir(lock, { recursive: true, mode: 0o700 });++ await expect(+ runMultiscan(+ options(+ paths,+ client(async (_repository, scanOptions = {}) =>+ completedScan(scanOptions.outputDir!),+ ),+ ),+ ),+ ).rejects.toThrow("A multiscan supervisor is already running.");+ expect((await lstat(lock)).isDirectory()).toBe(true);+ });++ test("recovers an interrupted stale-lock recovery claim", async () => {+ const paths = await fixture();+ const source = await repository(paths.root, "interrupted-recovery");+ await writeFile(+ paths.input,+ `id,repository,revision\ninterrupted,${source.path},${source.revision}\n`,+ );+ const lock = join(paths.output, ".lock");+ const recoveryPath = join(lock, ".recovering");+ await mkdir(lock, { recursive: true, mode: 0o700 });+ await writeFile(+ join(lock, "owner.json"),+ JSON.stringify({ pid: 999_999_999 }),+ {+ mode: 0o600,+ },+ );+ await writeFile(recoveryPath, "", { mode: 0o600 });+ const expired = new Date(Date.now() - 120_000);+ await utimes(recoveryPath, expired, expired);++ const summary = await runMultiscan(+ options(+ paths,+ client(async (_repository, scanOptions = {}) =>+ completedScan(scanOptions.outputDir!),+ ),+ ),+ );++ expect(summary).toMatchObject({ completed: 1, failed: 0 });+ await expect(access(lock)).rejects.toThrow();+ });++ test("allows only one supervisor to recover an abandoned lock", async () => {+ const paths = await fixture();+ const source = await repository(paths.root, "recovery-race");+ await writeFile(+ paths.input,+ `id,repository,revision\nrace,${source.path},${source.revision}\n`,+ );+ const lock = join(paths.output, ".lock");+ await mkdir(lock, { recursive: true, mode: 0o700 });+ await writeFile(+ join(lock, "owner.json"),+ JSON.stringify({ pid: 999_999_999 }),+ {+ mode: 0o600,+ },+ );+ let started!: () => void;+ let release!: () => void;+ const running = new Promise<void>((resolve) => {+ started = resolve;+ });+ const finish = new Promise<void>((resolve) => {+ release = resolve;+ });+ let active = 0;+ let maximum = 0;+ const security = client(async (_repository, scanOptions = {}) => {+ active += 1;+ maximum = Math.max(maximum, active);+ started();+ await finish;+ active -= 1;+ return completedScan(scanOptions.outputDir!);+ });+ const contenders = Promise.allSettled([+ runMultiscan(options(paths, security)),+ runMultiscan(options(paths, security)),+ ]);++ await running;+ release();+ const outcomes = await contenders;++ expect(maximum).toBe(1);+ expect(+ outcomes.filter((outcome) => outcome.status === "fulfilled"),+ ).toHaveLength(1);+ expect(+ outcomes.filter((outcome) => outcome.status === "rejected"),+ ).toHaveLength(1);+ });++ test("never removes a replacement owner's lock during interrupted cleanup", async () => {+ const paths = await fixture();+ const source = await repository(paths.root, "replacement-owner");+ await writeFile(+ paths.input,+ `id,repository,revision\nreplacement,${source.path},${source.revision}\n`,+ );+ let started!: () => void;+ let release!: () => void;+ const running = new Promise<void>((resolve) => {+ started = resolve;+ });+ const finish = new Promise<void>((resolve) => {+ release = resolve;+ });+ const first = runMultiscan(+ options(+ paths,+ client(async (_repository, scanOptions = {}) => {+ started();+ await finish;+ return completedScan(scanOptions.outputDir!);+ }),+ ),+ );+ await running;+ const lock = join(paths.output, ".lock");+ const abandoned = join(paths.output, ".lock.stale-interrupted");+ await rename(lock, abandoned);+ await mkdir(lock, { mode: 0o700 });+ const replacement = {+ pid: process.pid,+ ownerId: "replacement-supervisor",+ hostname: hostname(),+ processStartedAt: performance.timeOrigin,+ };+ await writeFile(join(lock, "owner.json"), JSON.stringify(replacement), {+ mode: 0o600,+ });++ release();+ await first;++ expect(+ JSON.parse(await readFile(join(lock, "owner.json"), "utf8")),+ ).toEqual(replacement);+ });++ test("never removes a replacement owner's lock when owner creation fails", async () => {+ const paths = await fixture();+ const source = await repository(paths.root, "owner-creation-race");+ await writeFile(+ paths.input,+ `id,repository,revision\nrace,${source.path},${source.revision}\n`,+ );+ const lock = join(paths.output, ".lock");+ const ownerPath = join(lock, "owner.json");+ const replacement = JSON.stringify({+ pid: process.pid,+ ownerId: "replacement-supervisor",+ hostname: hostname(),+ processStartedAt: performance.timeOrigin,+ });+ const originalWriteFile = filesystem.writeFile;+ const writeOwner = spyOn(filesystem, "writeFile").mockImplementation(+ async (path, data, options) => {+ if (String(path) !== ownerPath) {+ return await originalWriteFile(path, data, options);+ }+ writeOwner.mockRestore();+ await rename(lock, join(paths.output, ".lock.stale-owner-creation"));+ await mkdir(lock, { mode: 0o700 });+ await originalWriteFile(ownerPath, replacement, { mode: 0o600 });+ throw Object.assign(new Error("replacement already owns the lock"), {+ code: "EEXIST",+ });+ },+ );++ try {+ await expect(+ runMultiscan(+ options(+ paths,+ client(async (_repository, scanOptions = {}) =>+ completedScan(scanOptions.outputDir!),+ ),+ ),+ ),+ ).rejects.toThrow("replacement already owns the lock");+ expect(await readFile(ownerPath, "utf8")).toBe(replacement);+ } finally {+ writeOwner.mockRestore();+ }+ });+test("retries a failed attempt and records both durable receipts", async () => {const paths = await fixture();const source = await repository(paths.root, "retry");