Skip to content

OpenAI's Codex Security CLI and TypeScript SDK for finding, validating, and fixing security vulnerabilities. npm: https://www.npmjs.com/package/@openai/codex-security

40.0 KB
Raw Blame
import { codexWithRun, jsonCodex } from "./support/codex.js";
import { setFindingIdentity, sha256 } from "./support/finding-identity.js";
import { spawnSync } from "node:child_process";
import {
  chmod,
  mkdir,
  readFile,
  realpath,
  rename,
  rm,
  symlink,
  writeFile,
} from "node:fs/promises";
import { delimiter, dirname, join } from "node:path";
import { afterEach, expect, test } from "bun:test";
import {
  classifyScanDirectorySeverity,
  classifyScanSeverityInternal,
  readScanSeverityClassification,
} from "../src/classify-scan-severity.js";
import {
  classifySeverity,
  type ClassifySeverityOptions,
} from "../src/classify-severity.js";
import { Database } from "bun:sqlite";
import { loadContract } from "../src/contract.js";
import { workflowDigest } from "../src/finding-workflow.js";
import { defaultSeverityRubric } from "../src/default-severity-rubric.js";
import type { JsonObject } from "../src/config.js";
import type { Finding, FindingsDocument, ScanManifest } from "../src/models.js";
import { prepareScanPublication } from "../src/publication.js";
import { publishScanInternal } from "../src/publish.js";
import { resolvePluginPython, runWorkbench } from "../src/runtime.js";
import { copyCompletedScanFixture, PLUGIN_ROOT } from "./plugin-root.js";
import { createApiTestFixtures } from "./support/temporary-directories.js";

const { temporaryDirectory, cleanup } = createApiTestFixtures(
  "classify-scan-",
  false,
);
const destination = { destination: "linear", teamId: "team-example" } as const;
afterEach(cleanup);

async function fixture(scanId?: string, savedRoot?: string) {
  const root = savedRoot ?? (await temporaryDirectory());
  const scanDirectory = join(root, "scan");
  await copyCompletedScanFixture(scanDirectory);
  if (process.platform !== "win32") await chmod(scanDirectory, 0o700);
  const manifest = JSON.parse(
    await readFile(join(scanDirectory, "scan-manifest.json"), "utf8"),
  ) as ScanManifest;
  const document = JSON.parse(
    await readFile(join(scanDirectory, "findings.json"), "utf8"),
  ) as FindingsDocument;
  const other = structuredClone(document.findings[0]!);
  other.identity.instance = "second-instance";
  setFindingIdentity(manifest.scan, other);
  document.findings.push(other);
  if (scanId) {
    manifest.scan.id = scanId;
    document.scanId = scanId;
    for (const finding of document.findings)
      setFindingIdentity(manifest.scan, finding);
    const coveragePath = join(scanDirectory, "coverage.json");
    const coverage = JSON.parse(await readFile(coveragePath, "utf8"));
    coverage.scanId = scanId;
    await writeFile(coveragePath, JSON.stringify(coverage));
  }
  await writeFile(
    join(scanDirectory, "findings.json"),
    JSON.stringify(document),
  );
  for (const artifact of manifest.scan.artifacts)
    artifact.sha256 = sha256(
      await readFile(join(scanDirectory, artifact.path)),
    );
  await writeFile(
    join(scanDirectory, "scan-manifest.json"),
    JSON.stringify(manifest),
  );
  const rubricPath = join(root, "policy.md");
  await writeFile(
    rubricPath,
    "Assign Medium to bounded harm. Exclude administrative records.",
  );
  return {
    root,
    environment: {
      ...process.env,
      CODEX_SECURITY_STATE_DIR: join(root, "state"),
    },
    scanDirectory,
    rubricPath,
    findings: document.findings,
    scanId: manifest.scan.id,
  };
}

async function savedFixture() {
  const root = await temporaryDirectory();
  const repository = join(root, "repository");
  const caller = join(root, "caller");
  const scanDirectory = join(root, "scan");
  for (const directory of [repository, caller, scanDirectory])
    await mkdir(directory, { mode: 0o700 });
  const environment = {
    PATH: process.env["PATH"],
    SystemRoot: process.env["SystemRoot"],
    HOME: root,
    USERPROFILE: root,
    CODEX_HOME: join(root, "codex"),
    CODEX_SECURITY_STATE_DIR: join(root, "state"),
    CODEX_SECURITY_NO_UPDATE_NOTICE: "1",
  };
  const python = await resolvePluginPython({ environment });
  const workbench = { environment, python, pluginRoot: PLUGIN_ROOT };
  const registered = await runWorkbench(workbench, [
    "register-cli-scan",
    "--repository",
    repository,
    "--scan-dir",
    scanDirectory,
    "--recipe-json",
    JSON.stringify({
      config: {},
      mode: "standard",
      repository,
      target: { kind: "repository", paths: [] },
    }),
  ]);
  const scanId = registered["scanId"] as string;
  const artifacts = await fixture(scanId, root);
  const path = join(scanDirectory, "scan-manifest.json");
  const manifest = JSON.parse(await readFile(path, "utf8"));
  delete manifest.scan.sealedAt;
  delete manifest.scan.artifacts;
  manifest.scan.target.kind = "directory_snapshot";
  delete manifest.scan.target.revision;
  await writeFile(path, JSON.stringify(manifest));
  await runWorkbench(workbench, ["complete-scan", "--scan-id", scanId]);
  return { ...artifacts, environment, repository, caller, python };
}

function classifier(finding: Finding, excluded = false) {
  return jsonCodex(() => ({
    findingId: finding.findingId,
    decision: excluded ? "excluded" : "assessed",
    level: excluded ? null : "medium",
    rubricLabel: excluded ? null : "MEDIUM",
    rationale: excluded
      ? "Administrative record"
      : "Only bounded impact is established.",
    confidence: "high",
    reviewTrigger: null,
  }));
}

function query(environment: NodeJS.ProcessEnv, sql: string) {
  using database = new Database(
    join(environment["CODEX_SECURITY_STATE_DIR"]!, "workbench.sqlite3"),
  );
  database.run("PRAGMA foreign_keys = ON");
  return database.query<Record<string, unknown>, []>(sql).all();
}

test("classifies and reads a saved scan with an embedded NUL in its ID without Python", async () => {
  const { environment, root, scanDirectory, scanId, findings } =
    await fixture("imported\0scan");
  const withoutPython = {
    ...environment,
    PATH: dirname(Bun.which("node")!),
    PYTHON: join(root, "missing-python"),
  };
  const { scanId: classifiedScanId, ...classification } =
    await classifyScanDirectorySeverity(scanDirectory, {
      environment: withoutPython,
      codex: recordingClassifier().codex,
    });
  expect(classifiedScanId).toBe(scanId);
  expect(
    await readScanSeverityClassification(
      scanDirectory,
      scanId,
      findings,
      undefined,
      withoutPython,
    ),
  ).toEqual(classification);
});

test("saved scan artifacts cannot redirect knowledge-base reads", async () => {
  const { scanDirectory, root, environment, rubricPath } = await fixture();
  const alias = join(scanDirectory, "documents");
  await symlink(root, alias, "junction");
  await expect(
    classifyScanDirectorySeverity(scanDirectory, {
      environment,
      findingIds: [],
      rubricPath,
      knowledgeBasePaths: [join(alias, "policy.md")],
    }),
  ).rejects.toThrow(/symbolic link|directory link/i);
});

test("classifying a saved scan ID checks document links in its source directory", async () => {
  const { environment, repository, root, scanId } = await savedFixture();
  await symlink(root, join(repository, "documents"), "junction");
  const classify = (directory: string) =>
    classifyScanSeverityInternal(scanId, {
      environment,
      findingIds: [],
      rubricPath: join(directory, "documents", "policy.md"),
    });
  await expect(classify(repository)).rejects.toThrow(
    /symbolic link|directory link/i,
  );
  const moved = join(root, "moved");
  await rename(repository, moved);
  await expect(classify(moved)).rejects.toThrow("canonical path");
  await mkdir(repository);
  await expect(classify(moved)).rejects.toThrow("canonical path");
});

test("severity helpers skip Node executables in the caller's enclosing checkout", async () => {
  const { root, environment, scanDirectory } = await fixture();
  const repository = join(root, "repository");
  const bin = join(repository, "bin");
  const cwd = join(repository, "nested");
  await mkdir(bin, { recursive: true });
  await mkdir(cwd);
  await mkdir(join(repository, ".git"));
  const marker = join(root, "untrusted-node-ran");
  const node = join(bin, process.platform === "win32" ? "node.exe" : "node");
  await writeFile(
    node,
    process.platform === "win32"
      ? "synthetic invalid executable"
      : '#!/bin/sh\nprintf called > "$NODE_MARKER"\nexit 91\n',
    { mode: 0o700 },
  );
  const entryPoint = new URL(
    "../src/classify-scan-severity.ts",
    import.meta.url,
  );
  const result = spawnSync(
    process.execPath,
    [
      "--eval",
      `import { classifyScanDirectorySeverity } from ${JSON.stringify(entryPoint.href)};
await classifyScanDirectorySeverity(${JSON.stringify(scanDirectory)}, { environment: process.env, findingIds: [] });`,
    ],
    {
      cwd,
      encoding: "utf8",
      env: {
        ...environment,
        PATH: [bin, dirname(Bun.which("node")!)].join(delimiter),
        NODE_MARKER: marker,
      },
    },
  );
  expect(result.status, result.stderr).toBe(0);
  await expect(readFile(marker)).rejects.toMatchObject({ code: "ENOENT" });
});

for (const surface of ["SDK", "CLI"]) {
  test.skipIf(process.platform === "win32")(
    `saved severity ${surface} lookup rejects target Python before probing`,
    async () => {
      const f = await savedFixture();
      const marker = join(f.root, "python-probed");
      const executable = join(f.repository, "python3");
      await writeFile(
        executable,
        `#!/bin/sh\nprintf probed > '${marker.replaceAll("'", "'\\''")}'\nprintf "codex-security-python-ok\\n"\n`,
        { mode: 0o700 },
      );
      const python =
        surface === "CLI" ? join(f.root, "linked-python") : executable;
      if (surface === "CLI") await symlink(executable, python);
      const environment = {
        ...f.environment,
        PYTHON: python,
      };
      if (surface === "SDK") {
        const failure = await classifyScanSeverityInternal(
          f.scanId,
          { environment },
          { currentDirectory: () => f.caller },
        ).catch((error: unknown) => error);
        await expect(readFile(marker)).rejects.toMatchObject({
          code: "ENOENT",
        });
        expect(failure).toMatchObject({
          message: expect.stringContaining(
            "PYTHON interpreter is unavailable or unusable",
          ),
        });
      } else {
        const cli = new URL("../src/cli.ts", import.meta.url);
        const result = spawnSync(
          process.execPath,
          [
            "--eval",
            `const { main } = await import(${JSON.stringify(cli.href)});
process.exitCode = await main(["classify-severity", "--scan", ${JSON.stringify(f.scanId.slice(0, 8))}, "--json"]);`,
          ],
          { cwd: f.caller, encoding: "utf8", env: environment },
        );
        await expect(readFile(marker)).rejects.toMatchObject({
          code: "ENOENT",
        });
        expect(result.status, result.stderr).toBe(2);
        expect(result.stderr).toContain(
          "PYTHON interpreter is unavailable or unusable",
        );
      }
      await expect(
        readFile(join(f.scanDirectory, "severity-classification.json")),
      ).rejects.toMatchObject({ code: "ENOENT" });
    },
  );
}

test.each(["id", "prefix", "latest", "missing target"])(
  "saved severity supports %s without modifying sealed inputs or the environment",
  async (selection) => {
    const f = await savedFixture();
    const environment = Object.freeze({ ...f.environment, PYTHON: f.python });
    const before = await readFile(join(f.scanDirectory, "scan-manifest.json"));
    if (selection === "missing target")
      await rm(f.repository, { recursive: true });
    const result = await classifyScanSeverityInternal(
      selection === "latest"
        ? "latest"
        : selection === "prefix"
          ? f.scanId.slice(0, 8)
          : f.scanId,
      {
        environment,
        expectedScanId: f.scanId,
        rubricPath: await realpath(f.rubricPath),
        codex: recordingClassifier().codex,
      },
      {
        currentDirectory: () =>
          selection === "latest" ? f.repository : f.caller,
      },
    );
    expect(result.scanId).toBe(f.scanId);
    expect(result.assessments).toHaveLength(f.findings.length);
    expect(await readFile(join(f.scanDirectory, "scan-manifest.json"))).toEqual(
      before,
    );
    expect(environment).toEqual({ ...f.environment, PYTHON: f.python });
  },
);

test("saved severity protects repository and caller tools across concurrent lookups", async () => {
  const results = await Promise.allSettled(
    (["repository", "caller"] as const).map(async (location) => {
      const f = await savedFixture();
      const marker = join(f.root, "node-probed");
      await writeFile(
        join(f[location], process.platform === "win32" ? "node.exe" : "node"),
        process.platform === "win32"
          ? "synthetic invalid executable"
          : '#!/bin/sh\nprintf probed > "$NODE_MARKER"\nexit 91\n',
        { mode: 0o700 },
      );
      const result = await classifyScanSeverityInternal(
        f.scanId,
        {
          codex: recordingClassifier().codex,
          environment: {
            ...f.environment,
            PYTHON: f.python,
            PATH: [f[location], dirname(Bun.which("node")!)].join(delimiter),
            NODE_MARKER: marker,
          },
        },
        { currentDirectory: () => f.caller },
      );
      expect(result.scanId).toBe(f.scanId);
      expect(result.assessments).toHaveLength(f.findings.length);
      await expect(readFile(marker)).rejects.toMatchObject({ code: "ENOENT" });
    }),
  );
  for (const result of results)
    if (result.status === "rejected") throw result.reason;
});

function recordingClassifier() {
  const calls: string[] = [];
  const control = { failOn: "", failOnRun: 1, excluded: false };
  const codex = codexWithRun(async (prompt: string) => {
    const { finding } = JSON.parse(prompt.split("\n\n").at(-1)!) as {
      finding: Finding;
    };
    calls.push(finding.findingId);
    if (
      finding.findingId === control.failOn &&
      calls.filter((id) => id === finding.findingId).length ===
        control.failOnRun
    )
      throw new Error("Interrupted model call");
    return classifier(finding, control.excluded).startThread().run();
  });
  return { codex, calls, control };
}

test("checkpoints each finding, resumes missing work, and reprocesses only the selection", async () => {
  const { environment, scanDirectory, findings } = await fixture();
  const { codex, calls, control } = recordingClassifier();
  const options = { environment, codex };
  control.failOn = findings[1]!.findingId;
  control.failOnRun = 3;
  await expect(
    classifyScanDirectorySeverity(scanDirectory, options),
  ).rejects.toThrow("Interrupted model call");
  expect(
    query(environment, "SELECT finding_id FROM scan_severity_assessments").map(
      (row) => row["finding_id"],
    ),
  ).toEqual([findings[0]!.findingId]);
  await expect(
    prepareScanPublication(scanDirectory, { ...destination, environment }),
  ).rejects.toThrow("incomplete");
  control.failOn = "";
  calls.length = 0;
  const complete = await classifyScanDirectorySeverity(scanDirectory, options);
  expect(calls).toEqual(Array(3).fill(findings[1]!.findingId));
  expect(complete.assessments).toHaveLength(2);
  const { scanId: _scanId, ...assessment } = complete;
  expect(
    await readScanSeverityClassification(
      scanDirectory,
      complete.scanId,
      findings,
      undefined,
      environment,
    ),
  ).toEqual(assessment);
  const rows = query(
    environment,
    "SELECT * FROM scan_severity_assessments ORDER BY finding_id",
  );
  calls.length = 0;
  expect(
    (await classifyScanDirectorySeverity(scanDirectory, options)).assessments,
  ).toEqual(complete.assessments);
  expect(calls).toEqual([]);
  expect(
    query(
      environment,
      "SELECT * FROM scan_severity_assessments ORDER BY finding_id",
    ),
  ).toEqual(rows);

  control.excluded = true;
  const selected = {
    ...options,
    findingIds: [findings[0]!.findingId],
    reprocess: true,
  };
  const revised = await classifyScanDirectorySeverity(scanDirectory, selected);
  expect(calls).toEqual(Array(3).fill(findings[0]!.findingId));
  expect(revised.assessments[0]!.decision).toBe("excluded");
  const revisedRows = query(
    environment,
    "SELECT * FROM scan_severity_assessments ORDER BY finding_id",
  );
  expect(revisedRows).toHaveLength(2);
  expect(
    revisedRows.find((row) => row["finding_id"] === findings[1]!.findingId),
  ).toEqual(rows.find((row) => row["finding_id"] === findings[1]!.findingId));
  calls.length = 0;
  expect(
    (await classifyScanDirectorySeverity(scanDirectory, options))
      .assessments[0]!.decision,
  ).toBe("excluded");
  expect(calls).toEqual([]);
  expect(
    (
      await prepareScanPublication(scanDirectory, {
        ...destination,
        environment,
      })
    ).issues,
  ).toHaveLength(1);

  control.failOn = findings[0]!.findingId;
  control.failOnRun = 2;
  await expect(
    classifyScanDirectorySeverity(scanDirectory, selected),
  ).rejects.toThrow("Interrupted model call");
  expect(
    query(
      environment,
      "SELECT * FROM scan_severity_assessments ORDER BY finding_id",
    ),
  ).toEqual(revisedRows);
});

test.each(["same", "different"])(
  "preserves concurrent scan assessments with %s rubrics",
  async (rubric) => {
    const first = await fixture();
    const second = await fixture("scan_example_002");
    const environment = first.environment;
    if (rubric === "different")
      await writeFile(second.rubricPath, "Exclude administrative findings.");
    const firstModel = recordingClassifier();
    const secondModel = recordingClassifier();
    secondModel.control.excluded = true;
    const [firstResult, secondResult] = await Promise.all([
      classifyScanDirectorySeverity(first.scanDirectory, {
        environment,
        rubricPath: first.rubricPath,
        codex: firstModel.codex,
      }),
      classifyScanDirectorySeverity(second.scanDirectory, {
        environment,
        rubricPath: second.rubricPath,
        codex: secondModel.codex,
      }),
    ]);
    for (const [scan, result] of [
      [first, firstResult],
      [second, secondResult],
    ] as const) {
      const { scanId, ...classification } = result;
      expect(
        await readScanSeverityClassification(
          scan.scanDirectory,
          scanId,
          scan.findings,
          undefined,
          environment,
        ),
      ).toEqual(classification);
    }
    expect(
      (
        await prepareScanPublication(first.scanDirectory, {
          ...destination,
          environment,
        })
      ).issues.map((issue) => issue.priority),
    ).toEqual([3, 3]);
    expect(
      (
        await prepareScanPublication(second.scanDirectory, {
          ...destination,
          environment,
        })
      ).issues,
    ).toEqual([]);
    firstModel.calls.length = 0;
    expect(
      (
        await classifyScanDirectorySeverity(first.scanDirectory, {
          environment,
          rubricPath: first.rubricPath,
          codex: firstModel.codex,
        })
      ).assessments,
    ).toEqual(firstResult.assessments);
    expect(firstModel.calls).toEqual([]);
  },
);

test("migration leaves unindexed legacy assessments incomplete until reclassified", async () => {
  const first = await fixture();
  const second = await fixture("scan_example_002");
  const environment = first.environment;
  const { scanId, ...classification } = await classifyScanDirectorySeverity(
    first.scanDirectory,
    { environment, codex: recordingClassifier().codex },
  );
  query(environment, "DROP TABLE scan_severity_assessments");
  query(
    environment,
    "ALTER TABLE scan_severity_classifications DROP COLUMN rubric_binding_json",
  );
  for (const table of [
    "finding_severity_assessments",
    "scan_severity_classifications",
  ]) {
    query(environment, `ALTER TABLE ${table} DROP COLUMN classification_runs`);
  }
  query(
    environment,
    "DELETE FROM schema_migrations WHERE version IN (43, 52, 60)",
  );
  expect(
    await readScanSeverityClassification(
      first.scanDirectory,
      scanId,
      first.findings,
      undefined,
      environment,
    ),
  ).toEqual({
    ...classification,
    classificationRuns: undefined,
    rubricRevision: undefined,
    rubricSha256: workflowDigest([defaultSeverityRubric]),
  });
  expect(
    query(
      environment,
      "SELECT version FROM schema_migrations WHERE version = 43",
    ),
  ).toEqual([]);
  await classifyScanDirectorySeverity(second.scanDirectory, {
    environment,
    codex: recordingClassifier().codex,
  });
  await expect(
    readScanSeverityClassification(
      first.scanDirectory,
      scanId,
      first.findings,
      undefined,
      environment,
    ),
  ).rejects.toThrow("incomplete");
  expect(
    (
      await classifyScanDirectorySeverity(first.scanDirectory, {
        environment,
        codex: recordingClassifier().codex,
      })
    ).assessments,
  ).toEqual(classification.assessments);
  expect(
    query(
      environment,
      "SELECT version FROM schema_migrations WHERE version = 43",
    ),
  ).toEqual([{ version: 43 }]);
});

test("changed rubric, context, or evidence invalidates matching checkpoints", async () => {
  const { environment, root, scanDirectory, rubricPath, findings } =
    await fixture();
  const { codex, calls } = recordingClassifier();
  const options = { environment, rubricPath, codex };
  await classifyScanDirectorySeverity(scanDirectory, options);
  const originalFindings = query(
    environment,
    "SELECT * FROM findings ORDER BY id",
  );
  calls.length = 0;
  await writeFile(rubricPath, "Assign Medium to bounded metadata reads.");
  await classifyScanDirectorySeverity(scanDirectory, options);
  expect(calls).toHaveLength(6);
  const context = join(root, "context.md");
  await writeFile(context, "The system contains operational counters.");
  calls.length = 0;
  const withContext = { ...options, knowledgeBasePaths: [context] };
  await classifyScanDirectorySeverity(scanDirectory, withContext);
  expect(calls).toHaveLength(6);
  calls.length = 0;
  await writeFile(context, "The counters include protected metadata.");
  await classifyScanDirectorySeverity(scanDirectory, withContext);
  expect(calls).toHaveLength(6);

  const findingPath = join(scanDirectory, "findings.json");
  const document = JSON.parse(
    await readFile(findingPath, "utf8"),
  ) as FindingsDocument;
  document.findings[0]!.summary = "Additional evidence about the same finding.";
  await writeFile(findingPath, JSON.stringify(document));
  const manifestPath = join(scanDirectory, "scan-manifest.json");
  const manifest = JSON.parse(
    await readFile(manifestPath, "utf8"),
  ) as ScanManifest;
  for (const artifact of manifest.scan.artifacts)
    artifact.sha256 = sha256(
      await readFile(join(scanDirectory, artifact.path)),
    );
  await writeFile(manifestPath, JSON.stringify(manifest));
  await expect(
    prepareScanPublication(scanDirectory, { ...destination, environment }),
  ).rejects.toThrow("does not match");
  calls.length = 0;
  await classifyScanDirectorySeverity(scanDirectory, withContext);
  expect(calls).toEqual(Array(3).fill(findings[0]!.findingId));
  expect(query(environment, "SELECT * FROM findings ORDER BY id")).toEqual(
    originalFindings,
  );
});

test("classification refuses to store workflow state inside sealed scan artifacts", async () => {
  const { environment, scanDirectory } = await fixture();
  await expect(
    classifyScanDirectorySeverity(scanDirectory, {
      environment: {
        ...environment,
        CODEX_SECURITY_STATE_DIR: join(scanDirectory, "state"),
      },
    }),
  ).rejects.toThrow("outside");
});

test("a classified dedupe selection drives Linear priority and preserves sealed evidence", async () => {
  const { environment, scanDirectory, rubricPath, findings, scanId } =
    await fixture();
  const before = await loadContract(scanDirectory, { pluginRoot: PLUGIN_ROOT });
  const classification = await classifyScanDirectorySeverity(scanDirectory, {
    environment,
    findingIds: [findings[1]!.findingId],
    rubricPath,
    codex: classifier(findings[1]!),
  });
  expect(classification.scanId).toBe(scanId);
  expect(classification.assessments).toHaveLength(1);
  const result = await publishScanInternal(
    scanDirectory,
    {
      ...destination,
      dryRun: true,
    },
    { environment },
  );
  expect(result.issues).toHaveLength(1);
  expect(result.issues![0]).toMatchObject({
    findingId: findings[1]!.findingId,
    priority: 3,
  });
  expect(result.issues![0]!.title).toContain("[MEDIUM]");
  expect(result.issues![0]!.description).toContain("**Severity:** HIGH");
  expect(result.issues![0]!.description).toContain(
    "Only bounded impact is established.",
  );
  expect(
    await loadContract(scanDirectory, { pluginRoot: PLUGIN_ROOT }),
  ).toEqual(before);
  await expect(
    prepareScanPublication(scanDirectory, {
      environment,
      ...destination,
      findingIds: [findings[0]!.findingId],
    }),
  ).rejects.toThrow("missing from");
});

test("publication accepts in-memory assessments and exact ID selections", async () => {
  const { environment, scanDirectory, rubricPath, findings } = await fixture();
  const classification = await classifySeverity([findings[0]!], {
    rubricPath,
    codex: classifier(findings[0]!),
  });
  const prepared = await prepareScanPublication(scanDirectory, {
    environment,
    ...destination,
    classification,
  });
  expect(prepared.issues).toHaveLength(1);
  expect(prepared.issues[0]!.priority).toBe(3);
  expect(
    (
      await prepareScanPublication(scanDirectory, {
        environment,
        ...destination,
        findingIds: [findings[1]!.findingId],
      })
    ).issues[0]!.priority,
  ).toBe(2);
  expect(
    (
      await prepareScanPublication(scanDirectory, {
        ...destination,
        environment,
      })
    ).issues,
  ).toHaveLength(2);
  await expect(
    prepareScanPublication(scanDirectory, {
      environment,
      ...destination,
      findingIds: ["not-in-scan"],
    }),
  ).rejects.toThrow("belong");
});

test("exclusions and empty dedupe selections do not create tickets", async () => {
  const { environment, scanDirectory, rubricPath, findings } = await fixture();
  await classifyScanDirectorySeverity(scanDirectory, {
    environment,
    findingIds: [findings[0]!.findingId],
    rubricPath,
    codex: classifier(findings[0]!, true),
  });
  expect(
    (
      await prepareScanPublication(scanDirectory, {
        ...destination,
        environment,
      })
    ).issues,
  ).toEqual([]);
  await classifyScanDirectorySeverity(scanDirectory, {
    environment,
    findingIds: [],
  });
  expect(
    (
      await prepareScanPublication(scanDirectory, {
        ...destination,
        environment,
      })
    ).issues,
  ).toEqual([]);
});

test("failed or canceled reassessment leaves the last successful assessment intact", async () => {
  const { environment, scanDirectory, rubricPath, findings } = await fixture();
  await classifyScanDirectorySeverity(scanDirectory, {
    environment,
    codex: recordingClassifier().codex,
  });
  const path = join(scanDirectory, "severity-classification.json");
  const before = await readFile(path);
  await expect(
    classifyScanDirectorySeverity(scanDirectory, {
      environment,
      rubricPath,
      codex: classifier({ ...findings[0]!, findingId: "wrong" }),
    }),
  ).rejects.toThrow("invalid assessment");
  expect(await readFile(path)).toEqual(before);
  const controller = new AbortController();
  const codex: NonNullable<ClassifySeverityOptions["codex"]> = codexWithRun(
    async () => {
      controller.abort(new Error("stop"));
      return { finalResponse: "{}" };
    },
  );
  await expect(
    classifyScanDirectorySeverity(scanDirectory, {
      environment,
      rubricPath,
      codex,
      signal: controller.signal,
    }),
  ).rejects.toThrow();
  expect(await readFile(path)).toEqual(before);
});

test("publication reads SQLite even when the JSON export is modified or symlinked", async () => {
  const { environment, root, scanDirectory, findings } = await fixture();
  const result = await classifyScanDirectorySeverity(scanDirectory, {
    environment,
    codex: recordingClassifier().codex,
  });
  const path = join(scanDirectory, "severity-classification.json");
  await writeFile(path, JSON.stringify({ ...result, scanId: "other-scan" }));
  expect(
    (
      await prepareScanPublication(scanDirectory, {
        ...destination,
        environment,
      })
    ).issues,
  ).toHaveLength(findings.length);
  const stale = await classifySeverity(
    [{ ...findings[0]!, summary: "Different report" }],
    { codex: classifier(findings[0]!) },
  );
  await expect(
    prepareScanPublication(scanDirectory, {
      environment,
      ...destination,
      classification: stale,
    }),
  ).rejects.toThrow("does not match");
  await rm(path);
  const external = join(root, "outside.json");
  await writeFile(external, JSON.stringify(result));
  await symlink(external, path);
  expect(
    (
      await prepareScanPublication(scanDirectory, {
        ...destination,
        environment,
      })
    ).issues,
  ).toHaveLength(findings.length);
  await classifyScanDirectorySeverity(scanDirectory, {
    environment,
    codex: recordingClassifier().codex,
  });
  expect(await readFile(external, "utf8")).toBe(JSON.stringify(result));
});

test.each(["saved scan", "latest listing", "latest scan"])(
  "cancels a pending %s history request and preserves its cleanup and error",
  async (boundary) => {
    const controller = new AbortController();
    const reason = new Error("Classification canceled");
    const abortError = Object.assign(new Error("History request aborted"), {
      name: "AbortError",
      cause: reason,
    });
    const entered = Promise.withResolvers<void>();
    const response = Promise.withResolvers<JsonObject>();
    const calls: string[] = [];
    const events: string[] = [];
    let receivedSignal: AbortSignal | undefined;
    const operation = classifyScanSeverityInternal(
      boundary === "saved scan" ? "scan-example" : "latest",
      { signal: controller.signal },
      {
        currentDirectory: () => process.cwd(),
        runWorkbench: async (args, _input, signal) => {
          calls.push(args[0]!);
          if (boundary === "latest scan" && args[0] === "list-scans")
            return { scans: [{ scanId: "scan-example" }] };
          receivedSignal = signal;
          const abort = () => {
            events.push("aborted");
            response.reject(abortError);
          };
          signal?.addEventListener("abort", abort, { once: true });
          events.push("entered");
          entered.resolve();
          try {
            return await response.promise;
          } finally {
            signal?.removeEventListener("abort", abort);
            events.push("cleaned up");
          }
        },
      },
    );
    const pending = operation.catch((error: unknown) => error);
    try {
      await Promise.race([
        entered.promise,
        pending.then((error) => {
          throw error;
        }),
      ]);
      controller.abort(reason);
      response.reject(new Error("History lookup did not observe cancellation"));
      expect(await pending).toBe(abortError);
      expect(receivedSignal).toBe(controller.signal);
      expect(events).toEqual(["entered", "aborted", "cleaned up"]);
      expect(calls).toEqual(
        boundary === "saved scan"
          ? ["get-scan"]
          : boundary === "latest listing"
            ? ["list-scans"]
            : ["list-scans", "get-scan"],
      );
    } finally {
      response.resolve({});
      await pending;
    }
  },
);

test.each(["latest", "scan_prefix"])(
  "resolves %s using existing saved-scan history",
  async (selector) => {
    const { environment, scanDirectory, scanId } = await fixture();
    const seen: string[][] = [];
    const result = await classifyScanSeverityInternal(
      selector,
      { environment, codex: recordingClassifier().codex },
      {
        currentDirectory: () => scanDirectory,
        runWorkbench: async (args): Promise<JsonObject> => {
          seen.push([...args]);
          return args[0] === "list-scans"
            ? { scans: [{ scanId }] }
            : {
                scan: {
                  scanId,
                  scanDir: scanDirectory,
                  progress: { status: "complete" },
                },
              };
        },
      },
    );
    expect(result.scanId).toBe(scanId);
    expect(seen.at(-1)).toEqual([
      "get-scan",
      "--scan-id",
      selector === "latest" ? scanId : selector,
    ]);
    await expect(
      classifyScanDirectorySeverity(scanDirectory, {
        environment,
        expectedScanId: "other-scan",
      }),
    ).rejects.toThrow("do not match");
  },
);

test("migrates existing databases without changing findings and reads older state without writes", async () => {
  const { environment, scanDirectory } = await fixture();
  await classifyScanDirectorySeverity(scanDirectory, {
    environment,
    codex: recordingClassifier().codex,
  });
  const original = query(environment, "SELECT * FROM findings ORDER BY id");
  query(environment, "DROP TABLE scan_severity_assessments");
  query(environment, "DROP TABLE finding_severity_assessments");
  query(environment, "DROP TABLE scan_severity_classifications");
  query(
    environment,
    "DELETE FROM schema_migrations WHERE version IN (41, 43, 52, 60)",
  );
  expect(
    (
      await prepareScanPublication(scanDirectory, {
        ...destination,
        environment,
      })
    ).issues,
  ).toHaveLength(2);
  expect(
    query(
      environment,
      "SELECT version FROM schema_migrations WHERE version = 41",
    ),
  ).toEqual([]);
  await classifyScanDirectorySeverity(scanDirectory, {
    environment,
    codex: recordingClassifier().codex,
  });
  expect(
    query(
      environment,
      "SELECT version FROM schema_migrations WHERE version = 41",
    ),
  ).toEqual([{ version: 41 }]);
  expect(query(environment, "SELECT * FROM findings ORDER BY id")).toEqual(
    original,
  );
});

test("classifying another scan preserves both recurring-finding assessments", async () => {
  const first = await fixture();
  const second = await fixture();
  const manifestPath = join(second.scanDirectory, "scan-manifest.json");
  const manifest = JSON.parse(
    await readFile(manifestPath, "utf8"),
  ) as ScanManifest;
  manifest.scan.id = "scan_example_002";
  for (const file of ["findings.json", "coverage.json"]) {
    const path = join(second.scanDirectory, file);
    const document = JSON.parse(await readFile(path, "utf8"));
    document.scanId = manifest.scan.id;
    if (file === "findings.json") {
      for (const finding of document.findings as Finding[]) {
        finding.occurrenceId = `occ_${sha256([manifest.scan.id, finding.fingerprints.primary].join("\0")).slice(0, 24)}`;
      }
    }
    await writeFile(path, JSON.stringify(document));
  }
  for (const artifact of manifest.scan.artifacts)
    artifact.sha256 = sha256(
      await readFile(join(second.scanDirectory, artifact.path)),
    );
  await writeFile(manifestPath, JSON.stringify(manifest));
  const { codex, calls } = recordingClassifier();
  const options = {
    environment: first.environment,
    rubricPath: first.rubricPath,
    codex,
  };
  await classifyScanDirectorySeverity(first.scanDirectory, options);
  await classifyScanDirectorySeverity(second.scanDirectory, options);
  expect(calls).toHaveLength(12);
  calls.length = 0;
  for (const scan of [first, second, first, second]) {
    expect(
      (
        await prepareScanPublication(scan.scanDirectory, {
          ...destination,
          environment: first.environment,
        })
      ).issues,
    ).toHaveLength(2);
    await classifyScanDirectorySeverity(scan.scanDirectory, options);
  }
  expect(calls).toEqual([]);
});

test("default rubric replaces legacy inherited assessments and respects custom policy changes", async () => {
  const { environment, scanDirectory, rubricPath, findings, scanId } =
    await fixture();
  const { codex, calls } = recordingClassifier();
  const options = { environment, codex };
  await classifyScanDirectorySeverity(scanDirectory, options);
  // Recreate the provenance emitted by the former no-rubric inheritance path.
  for (const table of [
    "scan_severity_assessments",
    "finding_severity_assessments",
  ]) {
    query(
      environment,
      `UPDATE ${table} SET rubric_sha256 = NULL, source = 'existing-severity', level = 'high', rubric_label = NULL, confidence = NULL`,
    );
  }
  query(
    environment,
    "UPDATE scan_severity_classifications SET rubric_sha256 = NULL, rubric_binding_json = NULL",
  );
  const legacy = await readScanSeverityClassification(
    scanDirectory,
    scanId,
    findings,
    undefined,
    environment,
  );
  expect(
    legacy?.assessments.every((a) => a.source === "existing-severity"),
  ).toBe(true);
  calls.length = 0;
  const result = await classifyScanDirectorySeverity(scanDirectory, options);
  expect(calls).toHaveLength(6);
  expect(result.rubricSha256).toBe(sha256(defaultSeverityRubric));
  expect(result.rubricRevision).toBe(1);
  expect(
    result.assessments.every(
      (a) => a.source === "rubric" && a.level === "medium",
    ),
  ).toBe(true);
  expect(
    await readScanSeverityClassification(
      scanDirectory,
      scanId,
      findings,
      undefined,
      environment,
    ),
  ).toMatchObject({
    rubricSha256: result.rubricSha256,
    rubricRevision: result.rubricRevision,
  });
  calls.length = 0;
  query(
    environment,
    "UPDATE scan_severity_classifications SET rubric_binding_json = NULL",
  );
  expect(
    (await classifyScanDirectorySeverity(scanDirectory, options)).assessments,
  ).toEqual(result.assessments);
  expect(calls).toHaveLength(0);
  const custom = await classifyScanDirectorySeverity(scanDirectory, {
    ...options,
    rubricPath,
  });
  expect(calls).toHaveLength(6);
  expect(custom.rubricSha256).not.toBe(result.rubricSha256);
  calls.length = 0;
  expect(
    (await classifyScanDirectorySeverity(scanDirectory, options)).rubricSha256,
  ).toBe(result.rubricSha256);
  expect(calls).toHaveLength(6);
});

test("saved assessments bind their run count and invalidate changed or unrecorded counts", async () => {
  const { environment, scanDirectory, scanId, findings } = await fixture();
  const { codex, calls, control } = recordingClassifier();
  for (const [classificationRuns, expectedCalls] of [
    [3, 6],
    [3, 0],
    [1, 2],
    [1, 0],
    [10, 20],
    [3, 6],
  ]) {
    calls.length = 0;
    const result = await classifyScanDirectorySeverity(scanDirectory, {
      environment,
      codex,
      classificationRuns,
    });
    expect(calls).toHaveLength(expectedCalls!);
    expect(result.classificationRuns).toBe(classificationRuns);
    const saved = await readScanSeverityClassification(
      scanDirectory,
      scanId,
      findings,
      undefined,
      environment,
    );
    expect(saved?.classificationRuns).toBe(classificationRuns);
  }
  for (const table of [
    "scan_severity_classifications",
    "scan_severity_assessments",
    "finding_severity_assessments",
  ]) {
    query(environment, `UPDATE ${table} SET classification_runs = NULL`);
  }
  expect(
    (
      await readScanSeverityClassification(
        scanDirectory,
        scanId,
        findings,
        undefined,
        environment,
      )
    )?.classificationRuns,
  ).toBeUndefined();
  calls.length = 0;
  await classifyScanDirectorySeverity(scanDirectory, { environment, codex });
  expect(calls).toHaveLength(6);
  expect(
    query(
      environment,
      `SELECT classification_runs FROM scan_severity_classifications WHERE scan_id = '${scanId}'`,
    ),
  ).toEqual([{ classification_runs: 3 }]);
  calls.length = 0;
  control.failOn = findings[1]!.findingId;
  await expect(
    classifyScanDirectorySeverity(scanDirectory, {
      environment,
      codex,
      classificationRuns: 1,
    }),
  ).rejects.toThrow("Interrupted model call");
  await expect(
    readScanSeverityClassification(
      scanDirectory,
      scanId,
      findings,
      undefined,
      environment,
    ),
  ).rejects.toThrow("incomplete");
  calls.length = 0;
  control.failOn = "";
  await classifyScanDirectorySeverity(scanDirectory, {
    environment,
    codex,
    classificationRuns: 1,
  });
  expect(calls).toEqual([findings[1]!.findingId]);
});